Index: trunk/ippTools/src/Makefile.am =================================================================== --- trunk/ippTools/src/Makefile.am (revision 15536) +++ trunk/ippTools/src/Makefile.am (revision 15537) @@ -1,3 +1,4 @@ bin_PROGRAMS = \ + caltool \ camtool \ chiptool \ @@ -33,4 +34,5 @@ noinst_HEADERS = \ camtool.h \ + caltool.h \ chiptool.h \ detselect.h \ @@ -73,4 +75,10 @@ pztool.c \ pztoolConfig.c + +caltool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +caltool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +caltool_SOURCES = \ + caltool.c \ + caltoolConfig.c regtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) Index: trunk/ippTools/src/caltool.c =================================================================== --- trunk/ippTools/src/caltool.c (revision 15537) +++ trunk/ippTools/src/caltool.c (revision 15537) @@ -0,0 +1,6922 @@ +/* + * dettool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include + +#include "pxtools.h" +#include "dettool.h" + +static bool pendingMode(pxConfig *config); +static bool definebytagMode(pxConfig *config); +static bool definebyqueryMode(pxConfig *config); +static bool definebydetrunMode(pxConfig *config); +static bool runsMode(pxConfig *config); +static bool childlessrunMode(pxConfig *config); +static bool inputMode(pxConfig *config); +static bool rawMode(pxConfig *config); +// processedimfile +static bool toprocessedimfileMode(pxConfig *config); +static bool addprocessedimfileMode(pxConfig *config); +static bool processedimfileMode(pxConfig *config); +static bool revertprocessedimfileMode(pxConfig *config); +// processedexp +static bool toprocessedexpMode(pxConfig *config); +static bool addprocessedexpMode(pxConfig *config); +static bool processedexpMode(pxConfig *config); +static bool revertprocessedexpMode(pxConfig *config); +// stackedimfile +static bool tostackedMode(pxConfig *config); +static bool addstackedMode(pxConfig *config); +static bool stackedMode(pxConfig *config); +static bool revertstackedMode(pxConfig *config); +// normalizedstat +static bool tonormalizedstatMode(pxConfig *config); +static bool addnormalizedstatMode(pxConfig *config); +static bool normalizedstatMode(pxConfig *config); +static bool revertnormalizedstatMode(pxConfig *config); +// normalizedimfile +static bool tonormalizeMode(pxConfig *config); +static bool addnormalizedimfileMode(pxConfig *config); +static bool normalizedimfileMode(pxConfig *config); +static bool revertnormalizedimfileMode(pxConfig *config); +// normalizedexp +static bool tonormalizedexpMode(pxConfig *config); +static bool addnormalizedexpMode(pxConfig *config); +static bool normalizedexpMode(pxConfig *config); +static bool revertnormalizedexpMode(pxConfig *config); +// residimfile +static bool toresidimfileMode(pxConfig *config); +static bool addresidimfileMode(pxConfig *config); +static bool residimfileMode(pxConfig *config); +static bool revertresidimfileMode(pxConfig *config); +// residexp +static bool toresidexpMode(pxConfig *config); +static bool addresidexpMode(pxConfig *config); +static bool residexpMode(pxConfig *config); +static bool revertresidexpMode(pxConfig *config); +static bool updateresidexpMode(pxConfig *config); +// detrunsummary +static bool todetrunsummaryMode(pxConfig *config); +static bool adddetrunsummaryMode(pxConfig *config); +static bool detrunsummaryMode(pxConfig *config); +static bool revertdetrunsummaryMode(pxConfig *config); +static bool updatedetrunMode(pxConfig *config); +static bool rerunMode(pxConfig *config); +// register +static bool register_detrendMode(pxConfig *config); +static bool register_detrend_imfileMode(pxConfig *config); + +static detNormalizedStatImfileRow *detStackedToDetNormalizedStatImfile(pxConfig *config, detStackedImfileRow *stackedImfile); +static detNormalizedImfileRow *detNormalizedStatToDetNormalizedmfile(pxConfig *config, detNormalizedStatImfileRow *statImfile); +static detResidExpRow *mdToDetResidExp(pxConfig *config, psMetadata *row); +static detRunSummaryRow *mdToDetRunSummary(pxConfig *config, psMetadata *row); +//static psArray *validDetInputClassIds(pxConfig *config, const char *det_id); +//static psArray *searchInputImfiles(pxConfig *config, const char *det_id); +static detInputExpRow *rawDetrenTodetInputExpRow(rawExpRow *rawExp, psS64 det_id, psS32 iteration); +static psArray *searchRawImfiles(pxConfig *config, psMetadata *where); +static bool startNewIteration(pxConfig *config, psS64 det_id); +static psS32 incrementIteration(pxConfig *config, psS64 det_id); +static bool setDetRunState(pxConfig *config, psS64 det_id, const char *state); +static bool isValidMode(pxConfig *config, const char *mode); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = dettoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(DETTOOL_MODE_PENDING, pendingMode); + MODECASE(DETTOOL_MODE_DEFINEBYTAG, definebytagMode); + MODECASE(DETTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(DETTOOL_MODE_DEFINEBYDETRUN, definebydetrunMode); + MODECASE(DETTOOL_MODE_RUNS, runsMode); + MODECASE(DETTOOL_MODE_CHILDLESSRUN, childlessrunMode); + MODECASE(DETTOOL_MODE_INPUT, inputMode); + MODECASE(DETTOOL_MODE_RAW, rawMode); + // imfile + MODECASE(DETTOOL_MODE_TOPROCESSEDIMFILE,toprocessedimfileMode); + MODECASE(DETTOOL_MODE_ADDPROCESSEDIMFILE,addprocessedimfileMode); + MODECASE(DETTOOL_MODE_PROCESSEDIMFILE, processedimfileMode); + MODECASE(DETTOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileMode); + // exp + MODECASE(DETTOOL_MODE_TOPROCESSEDEXP, toprocessedexpMode); + MODECASE(DETTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpMode); + MODECASE(DETTOOL_MODE_PROCESSEDEXP, processedexpMode); + MODECASE(DETTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); + // stacked + MODECASE(DETTOOL_MODE_TOSTACKED, tostackedMode); + MODECASE(DETTOOL_MODE_ADDSTACKED, addstackedMode); + MODECASE(DETTOOL_MODE_STACKED, stackedMode); + MODECASE(DETTOOL_MODE_REVERTSTACKED, revertstackedMode); + // normalizedstat + MODECASE(DETTOOL_MODE_TONORMALIZEDSTAT, tonormalizedstatMode); + MODECASE(DETTOOL_MODE_ADDNORMALIZEDSTAT,addnormalizedstatMode); + MODECASE(DETTOOL_MODE_NORMALIZEDSTAT, normalizedstatMode); + MODECASE(DETTOOL_MODE_REVERTNORMALIZEDSTAT, revertnormalizedstatMode); + // normalizedimfile + MODECASE(DETTOOL_MODE_TONORMALIZE, tonormalizeMode); + MODECASE(DETTOOL_MODE_ADDNORMALIZEDIMFILE,addnormalizedimfileMode); + MODECASE(DETTOOL_MODE_NORMALIZEDIMFILE, normalizedimfileMode); + MODECASE(DETTOOL_MODE_REVERTNORMALIZEDIMFILE, revertnormalizedimfileMode); + // normalizedexp + MODECASE(DETTOOL_MODE_TONORMALIZEDEXP, tonormalizedexpMode); + MODECASE(DETTOOL_MODE_ADDNORMALIZEDEXP, addnormalizedexpMode); + MODECASE(DETTOOL_MODE_NORMALIZEDEXP, normalizedexpMode); + MODECASE(DETTOOL_MODE_REVERTNORMALIZEDEXP, revertnormalizedexpMode); + // residimfile + MODECASE(DETTOOL_MODE_TORESIDIMFILE, toresidimfileMode); + MODECASE(DETTOOL_MODE_ADDRESIDIMFILE, addresidimfileMode); + MODECASE(DETTOOL_MODE_RESIDIMFILE, residimfileMode); + MODECASE(DETTOOL_MODE_REVERTRESIDIMFILE,revertresidimfileMode); + // residexp + MODECASE(DETTOOL_MODE_TORESIDEXP, toresidexpMode); + MODECASE(DETTOOL_MODE_ADDRESIDEXP, addresidexpMode); + MODECASE(DETTOOL_MODE_RESIDEXP, residexpMode); + MODECASE(DETTOOL_MODE_REVERTRESIDEXP, revertresidexpMode); + MODECASE(DETTOOL_MODE_UPDATERESIDEXP, updateresidexpMode); + // detrunsummary + MODECASE(DETTOOL_MODE_TODETRUNSUMMARY, todetrunsummaryMode); + MODECASE(DETTOOL_MODE_ADDDETRUNSUMMARY, adddetrunsummaryMode); + MODECASE(DETTOOL_MODE_DETRUNSUMMARY, detrunsummaryMode); + MODECASE(DETTOOL_MODE_REVERTDETRUNSUMMARY, revertdetrunsummaryMode); + MODECASE(DETTOOL_MODE_UPDATEDETRUN, updatedetrunMode); + MODECASE(DETTOOL_MODE_RERUN, rerunMode); + // register + MODECASE(DETTOOL_MODE_REGISTER_DETREND, register_detrendMode); + MODECASE(DETTOOL_MODE_REGISTER_DETREND_IMFILE, register_detrend_imfileMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool pendingMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psString query = psStringCopy( + "SELECT" + " rawExp.*" + " FROM rawExp" + " LEFT JOIN detInputExp" + " ON rawExp.exp_id = detInputExp.exp_id" + " WHERE" + " detInputExp.exp_id IS NULL" + " AND rawExp.object != 'object'" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool definebytagMode(pxConfig *config) +{ + bool status = false; + + PS_ASSERT_PTR_NON_NULL(config, false); + + // what type of detRun is this? + // det_type is required + psString det_type = psMetadataLookupStr(&status, config->args, "-det_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_type"); + return false; + } + if (!det_type) { + psError(PS_ERR_UNKNOWN, true, "-det_type is required"); + return false; + } + + // optional + psString filelevel = psMetadataLookupStr(&status, config->args, "-filelevel"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filelevel"); + return false; + } + + psString workdir = psMetadataLookupStr(&status, config->args, "-workdir"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -workdir"); + return false; + } + if (!workdir) { + psError(PS_ERR_UNKNOWN, true, "-workdir is required"); + return false; + } + + // optional + psString mode = psMetadataLookupStr(&status, config->args, "-mode"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -mode"); + return false; + } + // check mode + if (mode && !isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalud mode"); + return false; + } + psString camera = psMetadataLookupStr(&status, config->args, "-inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst"); + return false; + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope"); + return false; + } + + psString exp_type = psMetadataLookupStr(&status, config->args, "-exp_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_type"); + return false; + } + + psString filter = psMetadataLookupStr(&status, config->args, "-filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter"); + return false; + } + + psF32 airmass_min = psMetadataLookupF32(&status, config->args, "-airmass_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass_min"); + return false; + } + + psF32 airmass_max = psMetadataLookupF32(&status, config->args, "-airmass_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass_max"); + return false; + } + + psF32 exp_time_min = psMetadataLookupF32(&status, config->args, "-exp_time_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time_min"); + return false; + } + + psF32 exp_time_max = psMetadataLookupF32(&status, config->args, "-exp_time_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time_max"); + return false; + } + + psF32 ccd_temp_min = psMetadataLookupF32(&status, config->args, "-ccd_temp_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp_min"); + return false; + } + + psF32 ccd_temp_max = psMetadataLookupF32(&status, config->args, "-ccd_temp_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp_max"); + return false; + } + + psF64 posang_min = psMetadataLookupF32(&status, config->args, "-posang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang_min"); + return false; + } + + psF64 posang_max = psMetadataLookupF32(&status, config->args, "-posang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang_max"); + return false; + } + psF64 solang_min = psMetadataLookupF32(&status, config->args, "-solang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -solang_min"); + return false; + } + + psF64 solang_max = psMetadataLookupF32(&status, config->args, "-solang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -solang_max"); + return false; + } + + psTime *registered = NULL; + { + psString registeredStr = psMetadataLookupStr(&status, config->args, "-registered"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -registered"); + return false; + } + // pass through NULL as this is an optional field + if (registeredStr) { + registered = psTimeFromISO(registeredStr, PS_TIME_UTC); + if (!registered) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", registeredStr); + return false; + } + } else { + registered = NULL; + } + } + + psTime *time_begin = NULL; + { + psString time_beginStr = psMetadataLookupStr(&status, config->args, "-time_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time_begin"); + return false; + } + // pass through NULL as this is an optional field + if (time_beginStr) { + time_begin = psTimeFromISO(time_beginStr, PS_TIME_UTC); + if (!time_begin) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", time_beginStr); + return false; + } + } else { + time_begin = NULL; + } + } + + psTime *time_end = NULL; + { + psString time_endStr = psMetadataLookupStr(&status, config->args, "-time_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time_end"); + return false; + } + // pass through NULL as this is an optional field + if (time_endStr) { + time_end = psTimeFromISO(time_endStr, PS_TIME_UTC); + if (!time_end) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", time_endStr); + return false; + } + } else { + time_end = NULL; + } + } + + psTime *use_begin = NULL; + { + psString use_beginStr = psMetadataLookupStr(&status, config->args, "-use_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -use_begin"); + return false; + } + // pass through NULL as this is an optional field + if (use_beginStr) { + use_begin = psTimeFromISO(use_beginStr, PS_TIME_UTC); + if (!use_begin) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", use_beginStr); + return false; + } + } else { + use_begin = NULL; + } + } + + psTime *use_end = NULL; + { + psString use_endStr = psMetadataLookupStr(&status, config->args, "-use_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -use_end"); + return false; + } + // pass through NULL as this is an optional field + if (use_endStr) { + use_end = psTimeFromISO(use_endStr, PS_TIME_UTC); + if (!use_end) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", use_endStr); + return false; + } + } else { + use_end = NULL; + } + } + + psString reduction = psMetadataLookupStr(&status, config->args, "-reduction"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -reduction"); + return false; + } + + psString label = psMetadataLookupStr(&status, config->args, "-label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label"); + return false; + } + + // we have to support multipe exp_ids + psMetadataItem *item = psMetadataLookup(config->args, "-exp_id"); + if (!item) { + // this shouldn't actually happen when using psArgs + psError(PS_ERR_UNKNOWN, true, "-exp_id is required"); + return false; + } + psMetadata *where = psMetadataAlloc(); + + // make sure that -exp_id was parsed correctly + // XXX this can be removed someday + if (item->type == PS_DATA_METADATA_MULTI) { + psListIterator *iter = psListIteratorAlloc(item->data.list, 0, false); + psMetadataItem *mItem = NULL; + while ((mItem = psListGetAndIncrement(iter))) { + psString exp_id = mItem->data.V; + // if exp_id is NULL then it means that -exp_id has not been + // specified + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, + "at least one -exp_id is required"); + psFree(where); + return false; + } + + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", + PS_META_DUPLICATE_OK, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(iter); + psFree(where); + return false; + } + } + psFree(iter); + } else { + psAbort( "-exp_id was not parsed correctly (this should not happen"); + } + + if (psListLength(where->list) < 1) { + psFree(where); + where = NULL; + } + + // check that the specified exp_ids actually exist + psArray *detrendExps = rawExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + if (!detrendExps) { + psError(PS_ERR_UNKNOWN, false, "no rawExp rows found"); + return false; + } + + // we should have one rawExp row per exp_id specified + if (psListLength(item->data.list) != psArrayLength(detrendExps)) { + psAbort( "an -exp_id matched more then one rawExp (this should not happen"); + + } + + // check to see if -filelevel was set on the command line + if (!filelevel) { + filelevel = psStringCopy(((rawExpRow *)detrendExps->data[0])->filelevel); + } + + // start a transaction so we don't end up with childlessed det_ids + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detrendExps); + return false; + } + + // the first iteration is always 0 + // XXX the camera name is set from the first inputExp + // XXX det_id + detRunInsert(config->dbh, + 0, + 0, + det_type, + mode, + "run", + filelevel, + workdir, + camera, + telescope, + exp_type, + reduction, + filter, + airmass_min, + airmass_max, + exp_time_min, + exp_time_max, + ccd_temp_min, + ccd_temp_max, + posang_min, + posang_max, + registered, + time_begin, + time_end, + use_begin, + use_end, + solang_min, + solang_max, + label, + 0 // parent + ); + psFree(registered); + psFree(time_begin); + psFree(time_end); + psFree(use_begin); + psFree(use_end); + psS64 det_id = psDBLastInsertID(config->dbh); + + // create new detInputExp row(s) from the rawExp row(s) + psArray *inputExps = psArrayAllocEmpty(psArrayLength(detrendExps)); + for (long i = 0; i < psArrayLength(detrendExps); i++) { + detInputExpRow *inputExp = rawDetrenTodetInputExpRow( + detrendExps->data[i], + det_id, + 0 // the first iteration is explicitly 0 + ); + psArrayAdd(inputExps, 0, inputExp); + psFree(inputExp); + } + + psFree(detrendExps); + + // insert detInputExp objects into the database + if (!detInputExpInsertObjects(config->dbh, inputExps)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(inputExps); + return false; + } + psFree(inputExps); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // print the new det_id + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id); + detRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check results + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 " (this should not happen)", det_id); + return false; + } + + if (!convertIdToStr(detRuns)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(detRuns); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, detRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detRuns); + return false; + } + psFree(detRuns); + + return true; +} + +#if 0 +// This function is used to convert the det_id from an int, as it is used +// internally, to be a string for external use. The rational being that we may +// want to change how det_id is generated in the future and don't want to +// external programs to become depending on this value being an int. +static bool convertDetIdToStr(psArray *mds) +{ + PS_ASSERT_PTR_NON_NULL(mds, false); + + for (long i = 0; i < psArrayLength(mds); i++) { + psMetadata *md = mds->data[i]; + bool status = false; + psS32 det_id = psMetadataLookupS32(&status, md, "det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for det_id"); + return false; + } + psMetadataRemoveKey(md, "det_id"); + psString det_idStr = psDBIntToString((psU64)det_id); + psMetadataAddStr(mds->data[i], PS_LIST_HEAD, "det_id", 0, NULL, det_idStr); + psFree(det_idStr); + } + + return true; +} +#endif + +static bool definebyqueryMode(pxConfig *config) +{ + bool status = false; + + PS_ASSERT_PTR_NON_NULL(config, false); + + // what type of detRun is this? + // det_type is required + psString det_type = psMetadataLookupStr(&status, config->args, "-det_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_type"); + return false; + } + if (!det_type) { + psError(PS_ERR_UNKNOWN, true, "-det_type is required"); + return false; + } + + // optional + psString filelevel = psMetadataLookupStr(&status, config->args, "-filelevel"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filelevel"); + return false; + } + + psString workdir = psMetadataLookupStr(&status, config->args, "-workdir"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -workdir"); + return false; + } + if (!workdir) { + psError(PS_ERR_UNKNOWN, true, "-workdir is required"); + return false; + } + psString camera = psMetadataLookupStr(&status, config->args, "-inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst"); + return false; + } + if (!camera) { + psError(PS_ERR_UNKNOWN, false, "-inst is required"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + { + bool status = false; + psString exp_type = psMetadataLookupStr(&status, config->args, "-select_exp_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_exp_type"); + return false; + } + if (exp_type) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_type", 0, "==", exp_type)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_type"); + psFree(where); + return false; + } + } + + // map -inst -> camera + psString camera = psMetadataLookupStr(&status, config->args, "-select_inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_inst"); + return false; + } + if (camera) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "camera", 0, "==", camera)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item inst"); + psFree(where); + return false; + } + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-select_telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_telescope"); + return false; + } + if (telescope) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "telescope", 0, "==", telescope)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item telescope"); + psFree(where); + return false; + } + } + + psString filter = psMetadataLookupStr(&status, config->args, "-select_filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_filter"); + return false; + } + if (filter) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "filter", 0, "==", filter)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item filter"); + psFree(where); + return false; + } + } + + psString uri = psMetadataLookupStr(&status, config->args, "-select_uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_uri"); + return false; + } + if (uri) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "uri", 0, "==", uri)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item uri"); + psFree(where); + return false; + } + } + + psString dateobs_begin = psMetadataLookupStr(&status, config->args, "-select_dateobs_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_dateobs_begin"); + psFree(where); + return false; + } + if (dateobs_begin) { + psTime *time = psTimeFromISO(dateobs_begin, PS_TIME_UTC); + if (!time) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", dateobs_begin); + psFree(time); + psFree(where); + return false; + } + if (!psMetadataAddTime(where, PS_LIST_TAIL, "dateobs", PS_META_DUPLICATE_OK, ">=", time)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item dateobs"); + psFree(time); + psFree(where); + return false; + } + psFree(time); + } + + psString dateobs_end = psMetadataLookupStr(&status, config->args, "-select_dateobs_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_dateobs_end"); + psFree(where); + return false; + } + if (dateobs_end) { + psTime *time = psTimeFromISO(dateobs_end, PS_TIME_UTC); + if (!time) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", dateobs_end); + psFree(time); + psFree(where); + return false; + } + if (!psMetadataAddTime(where, PS_LIST_TAIL, "dateobs", PS_META_DUPLICATE_OK, "<", time)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item dateobs"); + psFree(time); + psFree(where); + return false; + } + psFree(time); + } + + /** selection based on airmass range **/ + psF32 select_airmass_min = psMetadataLookupF32(&status, config->args, "-select_airmass_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_airmass_min"); + psFree(where); + return false; + } + if (isfinite(select_airmass_min)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "airmass", PS_META_DUPLICATE_OK, ">=", select_airmass_min)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item airmass"); + psFree(where); + return false; + } + } + psF32 select_airmass_max = psMetadataLookupF32(&status, config->args, "-select_airmass_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_airmass_max"); + psFree(where); + return false; + } + if (isfinite(select_airmass_max)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "airmass", PS_META_DUPLICATE_OK, "<=", select_airmass_max)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item airmass"); + psFree(where); + return false; + } + } + + psF32 select_sat_pixel_frac_max = psMetadataLookupF32(&status, config->args, "-select_sat_pixel_frac_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_sat_pixel_frac_max"); + psFree(where); + return false; + } + if (isfinite(select_sat_pixel_frac_max)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "sat_pixel_frac", PS_META_DUPLICATE_OK, "<=", select_sat_pixel_frac_max)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item sat_pixel_frac"); + psFree(where); + return false; + } + } + + /** selection based on exp_time range **/ + psF32 select_exp_time_min = psMetadataLookupF32(&status, config->args, "-select_exp_time_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_exp_time_min"); + psFree(where); + return false; + } + if (isfinite(select_exp_time_min)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "exp_time", PS_META_DUPLICATE_OK, ">=", select_exp_time_min)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_time"); + psFree(where); + return false; + } + } + psF32 select_exp_time_max = psMetadataLookupF32(&status, config->args, "-select_exp_time_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_exp_time_max"); + psFree(where); + return false; + } + if (isfinite(select_exp_time_max)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "exp_time", PS_META_DUPLICATE_OK, "<=", select_exp_time_max)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_time"); + psFree(where); + return false; + } + } + + /** selection based on ccd_temp range **/ + psF32 select_ccd_temp_min = psMetadataLookupF32(&status, config->args, "-select_ccd_temp_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_ccd_temp_min"); + psFree(where); + return false; + } + if (isfinite(select_ccd_temp_min)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "ccd_temp", PS_META_DUPLICATE_OK, ">=", select_ccd_temp_min)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item ccd_temp"); + psFree(where); + return false; + } + } + psF32 select_ccd_temp_max = psMetadataLookupF32(&status, config->args, "-select_ccd_temp_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_ccd_temp_max"); + psFree(where); + return false; + } + if (isfinite(select_ccd_temp_max)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "ccd_temp", PS_META_DUPLICATE_OK, "<=", select_ccd_temp_max)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item ccd_temp"); + psFree(where); + return false; + } + } + + /** selection based on posang **/ + psF32 select_posang_min = psMetadataLookupF32(&status, config->args, "-select_posang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_posang_min"); + psFree(where); + return false; + } + if (isfinite(select_posang_min)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "posang", PS_META_DUPLICATE_OK, ">=", select_posang_min)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item posang"); + psFree(where); + return false; + } + } + psF32 select_posang_max = psMetadataLookupF32(&status, config->args, "-select_posang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_posang_max"); + psFree(where); + return false; + } + if (isfinite(select_posang_max)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "posang", PS_META_DUPLICATE_OK, "<=", select_posang_max)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item posang"); + psFree(where); + return false; + } + } + + /** selection based on solang **/ + psF32 select_solang_min = psMetadataLookupF32(&status, config->args, "-select_solang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_solang_min"); + psFree(where); + return false; + } + if (isfinite(select_solang_min)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "solang", PS_META_DUPLICATE_OK, ">=", select_solang_min)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item solang"); + psFree(where); + return false; + } + } + psF32 select_solang_max = psMetadataLookupF32(&status, config->args, "-select_solang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_solang_max"); + psFree(where); + return false; + } + if (isfinite(select_solang_max)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "solang", PS_META_DUPLICATE_OK, "<=", select_solang_max)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item solang"); + psFree(where); + return false; + } + } + + } + + if (!psListLength(where->list)) { + psFree(where); + where = NULL; + } + + // there is some namespace overlap between the names of the fields we'd + // like to search by to setup a detrun and the names of the fields we'd + // like to assign values to so I've seperated them but prepending set- to + // the assigned values + + // optional + psString mode = psMetadataLookupStr(&status, config->args, "-mode"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -mode"); + return false; + } + // check mode + if (mode && !isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalud mode"); + return false; + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope"); + return false; + } + + psString filter = psMetadataLookupStr(&status, config->args, "-filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter"); + return false; + } + + psF32 airmass_min = psMetadataLookupF32(&status, config->args, "-airmass_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass_min"); + return false; + } + + psF32 airmass_max = psMetadataLookupF32(&status, config->args, "-airmass_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass_max"); + return false; + } + + psF32 exp_time_min = psMetadataLookupF32(&status, config->args, "-exp_time_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time_min"); + return false; + } + + psF32 exp_time_max = psMetadataLookupF32(&status, config->args, "-exp_time_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time_max"); + return false; + } + + psF32 ccd_temp_min = psMetadataLookupF32(&status, config->args, "-ccd_temp_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp_min"); + return false; + } + + psF32 ccd_temp_max = psMetadataLookupF32(&status, config->args, "-ccd_temp_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp_max"); + return false; + } + + psF64 posang_min = psMetadataLookupF32(&status, config->args, "-posang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang_min"); + return false; + } + + psF64 posang_max = psMetadataLookupF32(&status, config->args, "-posang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang_max"); + return false; + } + + psF64 solang_min = psMetadataLookupF32(&status, config->args, "-solang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -solang_min"); + return false; + } + + psF64 solang_max = psMetadataLookupF32(&status, config->args, "-solang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -solang_max"); + return false; + } + + psTime *registered = NULL; + { + psString registeredStr = psMetadataLookupStr(&status, config->args, "-registered"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -registered"); + return false; + } + // pass through NULL as this is an optional field + if (registeredStr) { + registered = psTimeFromISO(registeredStr, PS_TIME_UTC); + if (!registered) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", registeredStr); + return false; + } + } else { + registered = NULL; + } + } + + psTime *time_begin = NULL; + { + psString time_beginStr = psMetadataLookupStr(&status, config->args, "-time_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time_begin"); + return false; + } + // pass through NULL as this is an optional field + if (time_beginStr) { + time_begin = psTimeFromISO(time_beginStr, PS_TIME_UTC); + if (!time_begin) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", time_beginStr); + return false; + } + } else { + time_begin = NULL; + } + } + + psTime *time_end = NULL; + { + psString time_endStr = psMetadataLookupStr(&status, config->args, "-time_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time_end"); + return false; + } + // pass through NULL as this is an optional field + if (time_endStr) { + time_end = psTimeFromISO(time_endStr, PS_TIME_UTC); + if (!time_end) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", time_endStr); + return false; + } + } else { + time_end = NULL; + } + } + + psTime *use_begin = NULL; + { + psString use_beginStr = psMetadataLookupStr(&status, config->args, "-use_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -use_begin"); + return false; + } + // pass through NULL as this is an optional field + if (use_beginStr) { + use_begin = psTimeFromISO(use_beginStr, PS_TIME_UTC); + if (!use_begin) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", use_beginStr); + return false; + } + } else { + use_begin = NULL; + } + } + + psTime *use_end = NULL; + { + psString use_endStr = psMetadataLookupStr(&status, config->args, "-use_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -use_end"); + return false; + } + // pass through NULL as this is an optional field + if (use_endStr) { + use_end = psTimeFromISO(use_endStr, PS_TIME_UTC); + if (!use_end) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", use_endStr); + return false; + } + } else { + use_end = NULL; + } + } + + psString reduction = psMetadataLookupStr(&status, config->args, "-reduction"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -reduction"); + return false; + } + + psString label = psMetadataLookupStr(&status, config->args, "-label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label"); + return false; + } + + bool simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return false; + } + + bool pretend = psMetadataLookupBool(&status, config->args, "-pretend"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -pretend"); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return false; + } + + // search for rawExps with the specified options + psArray *detrendExps = rawExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + // make sure that we found at least one rawExp + if (!detrendExps) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return false; + } + if (!psArrayLength(detrendExps)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(detrendExps); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return true; + } + + // check to see if -filelevel was set on the command line + if (!filelevel) { + filelevel = psStringCopy(((rawExpRow *)detrendExps->data[0])->filelevel); + } + + if (pretend) { + // negative simple so the default is true + if (!rawExpPrintObjects(stdout, detrendExps, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detrendExps); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return false; + } + psFree(detrendExps); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return true; + } + + // start a transaction so we don't end up with childlessed det_ids + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detrendExps); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return false; + } + + // the first iteration is always 0 + // XXX det_id + detRunInsert(config->dbh, + 0, // det_id + 0, // iteration + det_type, + mode, + "run", // state + filelevel, + workdir, + camera, + telescope, + "NA", + reduction, + filter, + airmass_min, + airmass_max, + exp_time_min, + exp_time_max, + ccd_temp_min, + ccd_temp_max, + posang_min, + posang_max, + registered, + time_begin, + time_end, + use_begin, + use_end, + solang_min, + solang_max, + label, + 0 // parent + ); + psFree(registered); + psFree(time_begin); + psFree(time_end); + psFree(use_end); + psFree(use_begin); + psFree(use_end); + psS64 det_id = psDBLastInsertID(config->dbh); + + // create new detInputExp row(s) from the rawExp row(s) + psArray *inputExps = psArrayAllocEmpty(psArrayLength(detrendExps)); + for (long i = 0; i < psArrayLength(detrendExps); i++) { + detInputExpRow *inputExp = rawDetrenTodetInputExpRow( + detrendExps->data[i], + det_id, + 0 // the first iteration is explicitly 0 + ); + psArrayAdd(inputExps, 0, inputExp); + psFree(inputExp); + } + + psFree(detrendExps); + + // insert detInputExp objects into the database + if (!detInputExpInsertObjects(config->dbh, inputExps)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(inputExps); + return false; + } + psFree(inputExps); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + // print the new det_id + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id); + detRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check results + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 " (this should not happen)", det_id); + return false; + } + + // convert det_id to a string externaly + if (!convertIdToStr(detRuns)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert det_id to a string"); + psFree(detRuns); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, detRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detRuns); + return false; + } + psFree(detRuns); + + return true; +} + +static bool definebydetrunMode(pxConfig *config) +{ + bool status = false; + + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id is the only required input + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + + // lookup the detRun that we will be basing this one on + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", (psS64)atoll(det_id)); + detRuns = detRunSelectRowObjects(config->dbh, where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + // sanity check the result... we should have only found one det_id + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 " (this should not happen)", (psS64)atoll(det_id)); + return false; // unreachable + } + + // pull the detRun object out the result array + detRunRow *detRun = psMemIncrRefCounter(detRuns->data[0]); + + // discard the resultarray + psFree(detRuns); + + // set the det_id to 0/NULL so the database can assign it + detRun->det_id = 0; + + // reset the iteration to 0 + detRun->iteration = 0; + + // reset the state to "run" + psFree(detRun->state); + detRun->state = psStringCopy("run"); + + // walk through the optional values and update the detRun as required + psString det_type = psMetadataLookupStr(&status, config->args, "-set_det_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_det_type"); + return false; + } + if (det_type) { + psFree(detRun->det_type); + detRun->det_type = psStringCopy(det_type); + } + + psString mode = psMetadataLookupStr(&status, config->args, "-set_mode"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_mode"); + return false; + } + // check mode + if (mode && !isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalud mode"); + return false; + } + if (mode) { + psFree(detRun->mode); + detRun->mode = psStringCopy(mode); + } + + psString camera = psMetadataLookupStr(&status, config->args, "-set_inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_inst"); + return false; + } + if (camera) { + psFree(detRun->camera); + detRun->camera = psStringCopy(camera); + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-set_telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_telescope"); + return false; + } + if (telescope) { + psFree(detRun->telescope); + detRun->telescope = psStringCopy(telescope); + } + + psString exp_type = psMetadataLookupStr(&status, config->args, "-set_exp_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_exp_type"); + return false; + } + if (exp_type) { + psFree(detRun->exp_type); + detRun->exp_type = psStringCopy(exp_type); + } + + psString filelevel = psMetadataLookupStr(&status, config->args, "-set_filelevel"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_filelevel"); + return false; + } + if (filelevel) { + psFree(detRun->filelevel); + detRun->filelevel = psStringCopy(filelevel); + } + + psString workdir = psMetadataLookupStr(&status, config->args, "-set_workdir"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_workdir"); + return false; + } + if (workdir) { + psFree(detRun->workdir); + detRun->workdir = psStringCopy(workdir); + } + + psString filter = psMetadataLookupStr(&status, config->args, "-set_filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_filter"); + return false; + } + if (filter) { + psFree(detRun->filter); + detRun->filter = psStringCopy(filter); + } + + psF32 airmass_min = psMetadataLookupF32(&status, config->args, "-set_airmass_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_airmass_min"); + return false; + } + if (!isnan(airmass_min)) { + detRun->airmass_min = airmass_min; + } + + psF32 airmass_max = psMetadataLookupF32(&status, config->args, "-set_airmass_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_airmass_max"); + return false; + } + if (!isnan(airmass_max)) { + detRun->airmass_max = airmass_max; + } + + psF32 exp_time_min = psMetadataLookupF32(&status, config->args, "-set_exp_time_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_exp_time_min"); + return false; + } + if (!isnan(exp_time_min)) { + detRun->exp_time_min = exp_time_min; + } + + psF32 exp_time_max = psMetadataLookupF32(&status, config->args, "-set_exp_time_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_exp_time_max"); + return false; + } + if (!isnan(exp_time_max)) { + detRun->exp_time_max = exp_time_max; + } + + psF32 ccd_temp_min = psMetadataLookupF32(&status, config->args, "-set_ccd_temp_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_ccd_temp_min"); + return false; + } + if (!isnan(ccd_temp_min)) { + detRun->ccd_temp_min = ccd_temp_min; + } + + psF32 ccd_temp_max = psMetadataLookupF32(&status, config->args, "-set_ccd_temp_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_ccd_temp_max"); + return false; + } + if (!isnan(ccd_temp_max)) { + detRun->ccd_temp_max = ccd_temp_max; + } + + psF64 posang_min = psMetadataLookupF32(&status, config->args, "-set_posang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_posang_min"); + return false; + } + if (!isnan(posang_min)) { + detRun->posang_min = posang_min; + } + + psF64 posang_max = psMetadataLookupF32(&status, config->args, "-set_posang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_posang_max"); + return false; + } + if (!isnan(posang_max)) { + detRun->posang_max = posang_max; + } + + psF64 solang_min = psMetadataLookupF32(&status, config->args, "-set_solang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_solang_min"); + return false; + } + if (!isnan(solang_min)) { + detRun->solang_min = solang_min; + } + + psF64 solang_max = psMetadataLookupF32(&status, config->args, "-set_solang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_solang_max"); + return false; + } + if (!isnan(solang_max)) { + detRun->solang_max = solang_max; + } + + psString label = psMetadataLookupStr(&status, config->args, "-set_label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_label"); + return false; + } + if (label) { + detRun->label = label; + } + + psTime *registered = NULL; + { + psString registeredStr = psMetadataLookupStr(&status, config->args, "-set_registered"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_registered"); + return false; + } + // pass through NULL as this is an optional field + if (registeredStr) { + psFree(detRun->registered); + registered = psTimeFromISO(registeredStr, PS_TIME_UTC); + detRun->registered = psMemIncrRefCounter(registered); + psFree(registered); + } + } + + psTime *time_begin = NULL; + { + psString time_beginStr = psMetadataLookupStr(&status, config->args, "-set_time_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_time_begin"); + return false; + } + // pass through NULL as this is an optional field + if (time_beginStr) { + psFree(detRun->time_begin); + time_begin = psTimeFromISO(time_beginStr, PS_TIME_UTC); + detRun->time_begin = psMemIncrRefCounter(time_begin); + psFree(time_begin); + } + } + + psTime *time_end = NULL; + { + psString time_endStr = psMetadataLookupStr(&status, config->args, "-set_time_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_time_end"); + return false; + } + // pass through NULL as this is an optional field + if (time_endStr) { + psFree(detRun->time_end); + time_end = psTimeFromISO(time_endStr, PS_TIME_UTC); + detRun->time_end = psMemIncrRefCounter(time_end); + psFree(time_end); + } + } + + psTime *use_begin = NULL; + { + psString use_beginStr = psMetadataLookupStr(&status, config->args, "-set_use_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_use_begin"); + return false; + } + // pass through NULL as this is an optional field + if (use_beginStr) { + psFree(detRun->use_begin); + use_begin = psTimeFromISO(use_beginStr, PS_TIME_UTC); + detRun->use_begin = psMemIncrRefCounter(use_begin); + psFree(use_begin); + } + } + + psTime *use_end = NULL; + { + psString use_endStr = psMetadataLookupStr(&status, config->args, "-set_use_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_use_end"); + return false; + } + // pass through NULL as this is an optional field + if (use_endStr) { + psFree(detRun->use_end); + use_end = psTimeFromISO(use_endStr, PS_TIME_UTC); + detRun->use_end = psMemIncrRefCounter(use_end); + psFree(use_end); + } + } + + psString reduction = psMetadataLookupStr(&status, config->args, "-set_reduction"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_reduction"); + return false; + } + if (reduction) { + psFree(detRun->reduction); + detRun->reduction = psStringCopy(reduction); + } + + // create a metadata to restrict detInputExp's be in in the specified range + + psMetadata *time_filter = psMetadataAlloc(); + + { + psString timeStr = psMetadataLookupStr(&status, config->args, "-filter_input_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter_input_begin"); + return false; + } + // pass through NULL as this is an optional field + if (timeStr) { + psTime *time = psTimeFromISO(timeStr, PS_TIME_UTC); + if (!psMetadataAddTime(time_filter, PS_LIST_TAIL, "dateobs", PS_META_DUPLICATE_OK, ">=", time)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item dateobs"); + psFree(time); + psFree(time_filter); + return false; + } + psFree(time); + } + } + + { + psString timeStr = psMetadataLookupStr(&status, config->args, "-filter_input_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter_input_end"); + return false; + } + // pass through NULL as this is an optional field + if (timeStr) { + psTime *time = psTimeFromISO(timeStr, PS_TIME_UTC); + if (!psMetadataAddTime(time_filter, PS_LIST_TAIL, "dateobs", PS_META_DUPLICATE_OK, "<", time)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item dateobs"); + psFree(time); + psFree(time_filter); + return false; + } + psFree(time); + } + } + + + // start a transaction so we don't end up with childlessed det_ids + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(time_filter); + psFree(detRun); + return false; + } + + if (!detRunInsertObject(config->dbh, detRun)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(time_filter); + psFree(detRun); + return false; + } + psFree(detRun); + + // get the det_id + psS64 newDet_id = psDBLastInsertID(config->dbh); + + psString query = psStringCopy( + "INSERT INTO detInputExp" + " SELECT" + " %d," + " 0," + " detResidExp.exp_id," + " detResidExp.accept" + " FROM detResidExp" + " JOIN rawExp" + " USING(exp_id)" + " WHERE det_id = %d" + ); + + if (time_filter->list->n) { + psString whereClause = psDBGenerateWhereConditionSQL(time_filter, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(time_filter); + + if (!p_psDBRunQuery(config->dbh, query, (psS64)newDet_id, (psS64)atoll(det_id))) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // print the new det_id + psArray *newDetRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", newDet_id); + newDetRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!newDetRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check the result... we should have only found one det_id + if (psArrayLength(newDetRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 " (this should not happen)", newDet_id); + return false; // unreachable + } + + // convert det_id to a string externaly + if (!convertIdToStr(detRuns)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(detRuns); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, newDetRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(newDetRuns); + return false; + } + psFree(newDetRuns); + + return true; +} + +static bool runsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // XXX fix the hard coding of the table name + psArray *runs = psDBSelectRows(config->dbh, "detRun", config->where, 0); + if (!runs) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(runs)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(runs); + return true; + } + + // convert det_id to a string externaly + if (!convertIdToStr(runs)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(runs); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, runs, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(runs); + return false; + } + + psFree(runs); + + return true; +} + +static bool childlessrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = psStringCopy( + "SELECT DISTINCT\n" + " detRun.*\n" + " FROM detRun\n" + " LEFT JOIN detRun as foo\n" + " ON foo.parent = detRun.det_id\n" + " WHERE\n" + " detRun.state = 'stop'\n" + " AND detRun.mode = 'master'\n" + " AND foo.det_id IS NULL\n" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detRun"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // convert det_id to a string externaly + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static detInputExpRow *rawDetrenTodetInputExpRow(rawExpRow *rawExp, psS64 det_id, psS32 iteration) +{ + PS_ASSERT_PTR_NON_NULL(rawExp, NULL); + + return detInputExpRowAlloc( + det_id, + iteration, + rawExp->exp_id, + true // use + ); +} + +static bool inputMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // select detInputExp.* + // select rawExp.* + // by: + // exp_id + + psString query = psStringCopy( + "SELECT DISTINCT *" + " FROM detInputExp" + " JOIN rawExp" + " USING(exp_id)" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereSQL(config->where, "detInputExp"); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detInputExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool rawMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_raw.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool toprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("dettool_toprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingProcessedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static psArray *searchRawImfiles(pxConfig *config, psMetadata *where) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // use the default where if "where" is NULL + if (!where) { + where = config->where; + } + + // select exp_ids from detInputExp matching det_idp + // where query should be pre-generated + psArray *detInputExp = + detInputExpSelectRowObjects(config->dbh, where, 0); + if (!detInputExp) { + psError(PS_ERR_UNKNOWN, false, "no rawExp rows found"); + return NULL; + } + + // generate where query with just the exp_ids + psMetadata *where_exp_ids = psMetadataAlloc(); + for (long i = 0; i < psArrayLength(detInputExp); i++) { + detInputExpRow *row = detInputExp->data[i]; + if (!psMetadataAddS64(where_exp_ids, PS_LIST_TAIL, "exp_id", + PS_META_DUPLICATE_OK, "==", row->exp_id) + ) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(detInputExp); + psFree(where_exp_ids); + return NULL; + } + } + psFree(detInputExp); + + // select rawImfiles with matching exp_ids + psArray *rawImfiles = + rawImfileSelectRowObjects(config->dbh, where_exp_ids, 0); + psFree(where_exp_ids); + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found"); + return NULL; + } + + return rawImfiles; +} + +static bool addprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id, exp_id, class_id, uri, recipe, -bg, -bg_stdev + // are required + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id"); + return false; + } + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "-exp_id is required"); + return false; + } + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + return false; + } + if (!class_id) { + psError(PS_ERR_UNKNOWN, true, "-class_id is required"); + return false; + } + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + return false; + } + if (!recipe) { + psError(PS_ERR_UNKNOWN, true, "-recip is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + //psError(PS_ERR_UNKNOWN, true, "-bg is required"); + //return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + + // optional + psF64 fringe_0 = psMetadataLookupF64(&status, config->args, "-fringe_0"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_0"); + return false; + } + psF64 fringe_1 = psMetadataLookupF64(&status, config->args, "-fringe_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_1"); + return false; + } + psF64 fringe_2 = psMetadataLookupF64(&status, config->args, "-fringe_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_2"); + return false; + } + + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + // find the matching rawImfile by exp_id/class_id + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", 0, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + return false; + } + if (!psMetadataAddStr(where, PS_LIST_TAIL, "class_id", 0, "==", class_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class_id"); + psFree(where); + return false; + } + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + psArray *rawImfiles = rawImfileSelectRowObjects(config->dbh, where, 0); + psFree(where); + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found "); + return false; + } + + + // create a new detProcessedImfile object + detProcessedImfileRow *detRow = detProcessedImfileRowAlloc( + (psS64)atoll(det_id), + (psS64)atoll(exp_id), + class_id, + uri, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + fringe_0, + fringe_1, + fringe_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + code + ); + psFree(rawImfiles); + + // insert the new row into the detProcessedImfile table + if (!detProcessedImfileInsertObject(config->dbh, detRow)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detRow); + return false; + } + + psFree(detRow); + + return true; +} + +static bool toprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("dettool_toprocessedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id, exp_id, recip, -bg, -bg_stdev + // are required + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id"); + return false; + } + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "-exp_id is required"); + return false; + } + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + return false; + } + if (!recipe) { + psError(PS_ERR_UNKNOWN, true, "-recip is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + // optional + psF64 fringe_0 = psMetadataLookupF64(&status, config->args, "-fringe_0"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_0"); + return false; + } + psF64 fringe_1 = psMetadataLookupF64(&status, config->args, "-fringe_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_1"); + return false; + } + psF64 fringe_2 = psMetadataLookupF64(&status, config->args, "-fringe_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_2"); + return false; + } + + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + psString query = psStringCopy( + " SELECT DISTINCT" + " detProcessedImfile.det_id," + " detRun.iteration," + " detRun.det_type," + " detProcessedImfile.exp_id" + " FROM detRun" + " JOIN detInputExp" + " ON detRun.det_id = detInputExp.det_id" + " AND detRun.iteration = detInputExp.iteration" + " JOIN rawExp" + " ON detInputExp.exp_id = rawExp.exp_id" + " JOIN detProcessedImfile" + " ON detInputExp.det_id = detProcessedImfile.det_id" + " AND detInputExp.exp_id = detProcessedImfile.exp_id" + " LEFT JOIN detProcessedExp" + " ON detInputExp.det_id = detProcessedExp.det_id" + " AND detProcessedImfile.exp_id= detProcessedExp.exp_id" + " LEFT JOIN rawImfile" + " ON detInputExp.exp_id = rawImfile.exp_id" + " AND detProcessedImfile.class_id = rawImfile.class_id" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + " AND detProcessedExp.det_id IS NULL" + " AND detProcessedExp.exp_id IS NULL" + " AND detInputExp.include = 1" + " AND detRun.det_id = %s" + " AND detProcessedImfile.exp_id = '%s'" + " GROUP BY" + " detProcessedImfile.class_id," + " rawImfile.class_id," + " detRun.det_id" + " HAVING" + " COUNT(detProcessedImfile.class_id) = COUNT(rawImfile.class_id)" + ); + + if (!p_psDBRunQuery(config->dbh, query, det_id, exp_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + psFree(output); + + // create a new detProcessedImfile object + detProcessedExpRow *detRow = detProcessedExpRowAlloc( + (psS64)atoll(det_id), + (psS64)atoll(exp_id), + recipe, + bg, + bg_stdev, + bg_mean_stdev, + fringe_0, + fringe_1, + fringe_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + code + ); + + // insert the new row into the detProcessedImfile table + if (!detProcessedExpInsertObject(config->dbh, detRow)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detRow); + return false; + } + + psFree(detRow); + + return true; +} + +static bool processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = psStringCopy("SELECT * FROM detProcessedExp"); + + if (config->where) { + psString whereClause = psDBGenerateWhereSQL(config->where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detProcessedExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detProcessedExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_revertprocessedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detProcessedExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool tostackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("dettool_tostacked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingStackedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + char *value = NULL; + bool status = false; + + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = psStringCopy( + " SELECT DISTINCT" + " detRun.det_type," + " rawExp.exp_time," + " detProcessedImfile.*" + " FROM detProcessedImfile" + " JOIN detRun" + " USING(det_id)" + " JOIN detInputExp" + " ON detRun.det_id = detInputExp.det_id" + " AND detRun.iteration = detInputExp.iteration" + " AND detProcessedImfile.exp_id = detInputExp.exp_id" + " JOIN rawExp" + " ON rawExp.exp_id = detProcessedImfile.exp_id" + " WHERE" + ); + // NOTE the above WHERE is completed with the following line: + + // add the two required restrictions: detRun.state and detRun.mode + if ((value = psMetadataLookupStr(&status, config->args, "-select_state"))) { + psStringAppend(&query, " detRun.state = '%s'", value); + } else { + psStringAppend(&query, " detRun.state = 'run'"); + } + if ((value = psMetadataLookupStr(&status, config->args, "-select_mode"))) { + psStringAppend(&query, " AND detRun.mode = '%s'", value); + } else { + psStringAppend(&query, " AND detRun.mode = 'master'"); + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detProcessedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + { + bool status = false; + bool included = psMetadataLookupBool(&status, config->args, "-included"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + // restrict search to included imfiles + if (included) { + psStringAppend(&query, " AND detInputExp.include = 1"); + } + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detProcessedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detProcessedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_revertprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detProcessedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool addstackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id, class_id, uri, & recipe are required + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + return false; + } + if (!class_id) { + psError(PS_ERR_UNKNOWN, true, "-class_id is required"); + return false; + } + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + return false; + } + if (!recipe) { + psError(PS_ERR_UNKNOWN, true, "-recip is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + + // optional values + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + // default values + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + return false; + } + + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + // correlate the class_id against the input exposure(s) + + // we have to generate our own where clause as we want to search only by the + // det_id + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", + (psS64)atoll(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + return false; + } + + psArray *rawImfiles = searchRawImfiles(config, where); + psFree(where); + + bool valid_class_id = false; + if (rawImfiles) { + for (long i = 0; i < psArrayLength(rawImfiles); i++) { + if (strcmp(class_id, ((rawImfileRow *)rawImfiles->data[i])->class_id) == 0) { + valid_class_id = true; + break; + } + } + psFree(rawImfiles); + } + + if (!valid_class_id) { + psError(PS_ERR_UNKNOWN, true, + "class_id can not be correlated with the input exposures"); + return false; + } + + // create a new detStackedImfile object + detStackedImfileRow *stackedImfile = detStackedImfileRowAlloc( + (psS64)atoll(det_id), + iteration, + class_id, + uri, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + code + ); + + // insert the new row into the detProcessedImfile table + if (!detStackedImfileInsertObject(config->dbh, stackedImfile)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(stackedImfile); + return false; + } + + psFree(stackedImfile); + + return true; +} + +static bool stackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + // select detStackedImfile.* + // by: + // where det_id, iteration, class_id is not in detNormalizedImfile + + psString query = psStringCopy( + "SELECT" + " detStackedImfile.*" + " FROM detStackedImfile" + " JOIN detRun" + " USING(det_id, iteration)" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detStackedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detStackedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detStackedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertstackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_revertstacked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detStackedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +static bool tonormalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("dettool_tonormalizedstat.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingNormStatImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addnormalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // select * from detStackedImfile + // by det_id, iteration, class_id + // where det_id, iteration, class_id is not in detNormalizedStatImfile + psString query = psStringCopy( + "SELECT DISTINCT" + " detStackedImfile.*" + " FROM detStackedImfile" + " LEFT JOIN detNormalizedStatImfile" + " USING(det_id, iteration, class_id)" + " WHERE" + " detNormalizedStatImfile.det_id IS NULL" + " AND detNormalizedStatImfile.iteration IS NULL" + " AND detNormalizedStatImfile.class_id IS NULL" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detStackedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + // convert metadata into a detStackedImfile object + detStackedImfileRow *stackedImfile = detStackedImfileObjectFromMetadata(row); + // convert detStackedImfile object into a detNormalizedStat object + detNormalizedStatImfileRow *stat = detStackedToDetNormalizedStatImfile(config, stackedImfile); + psFree(stackedImfile); + if (!stat) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to convert detStackedImfile to detNormalizedStatImfile"); + psFree(output); + return false; + } + // insert detNormlized Stat object into the database + if (!detNormalizedStatImfileInsertObject(config->dbh, stat)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(stat); + psFree(output); + } + psFree(stat); + } + + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static detNormalizedStatImfileRow *detStackedToDetNormalizedStatImfile(pxConfig *config, detStackedImfileRow *stackedImfile) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(stackedImfile, NULL); + + bool status = false; + psF32 norm = psMetadataLookupF32(&status, config->args, "-norm"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -norm"); + return false; + } + //if (isnan(norm)) { + // psError(PS_ERR_UNKNOWN, true, "-norm is required"); + // return false; + //} + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + return detNormalizedStatImfileRowAlloc( + stackedImfile->det_id, + stackedImfile->iteration, + stackedImfile->class_id, + norm, + code + ); +} + + +static bool normalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = pxDataGet("dettool_normalizedstat.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "WHERE fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "WHERE fault = 0"); + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detNormalizedStatImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool revertnormalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_revertnormalizedstat.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detNormalizedStatImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +static bool tonormalizeMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + // select detNormalizedStatImfile.* + // by: + // where det_id, iteration, class_id is not in detNormalizedImfile + + psString query = psStringCopy( + "SELECT DISTINCT" + " detRun.det_type," + " detRun.workdir," + " rawExp.camera," + " detStackedImfile.uri," + " detNormalizedStatImfile.*" + " FROM detRun" + " JOIN detStackedImfile" + " USING(det_id, iteration)" + " JOIN detInputExp" + " USING(det_id, iteration)" + " JOIN rawExp" + " ON detInputExp.exp_id = rawExp.exp_id" + " JOIN detNormalizedStatImfile" + " ON detStackedImfile.det_id = detNormalizedStatImfile.det_id" + " AND detStackedImfile.iteration = detNormalizedStatImfile.iteration" + " AND detStackedImfile.class_id = detNormalizedStatImfile.class_id" + " LEFT JOIN detNormalizedImfile" + " ON detNormalizedStatImfile.det_id = detNormalizedImfile.det_id" + " AND detNormalizedStatImfile.iteration = detNormalizedImfile.iteration" + " AND detNormalizedStatImfile.class_id = detNormalizedImfile.class_id" + " WHERE" + " detNormalizedImfile.det_id IS NULL" + " AND detNormalizedImfile.iteration IS NULL" + " AND detNormalizedImfile.class_id IS NULL" + " AND detNormalizedStatImfile.fault = 0" + ); + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingNormImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +#if 0 +// XXX this function was left in commented as this method may be useful in the +// future +static psArray *validDetInputClassIds(pxConfig *config, const char *det_id) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + // det_id is input as a string because the fact that it is an integer + // is just a database impliementation detail. + PS_ASSERT_PTR_NON_NULL(det_id, NULL); + + psArray *rawImfiles = searchInputImfiles(config, det_id); + if (!rawImfiles) { + return NULL; + } + + psArray *valid_class_ids = NULL; + { + // All this jumping through hoops is so that we end up with unique + // values. PP thinks that making multiple passes through this array and + // deleting matched elements would end up being more expensive then a + // double sort and stagger scheme. JH thinks it would be cheaper to + // just do a unqiue sort and delete. So this is really just a cheap + // hack to avoid implimenting a unique sort function but at least it's + // stable. + psHash *hash = psHashAlloc(psArrayLength(rawImfiles)); + for (long i = 0; i < psArrayLength(rawImfiles); i++) { + psHashAdd(hash, + ((rawImfileRow *)rawImfiles->data[i])->class_id, + ((rawImfileRow *)rawImfiles->data[i])->class_id + ); + } + valid_class_ids = psHashToArray(hash); + psFree(hash); + } + psFree(rawImfiles); + + return valid_class_ids; +} + +static psArray *searchInputImfiles(pxConfig *config, const char *det_id) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + // det_id is input as a string because the fact that it is an integer + // is just a database impliementation detail. + PS_ASSERT_PTR_NON_NULL(det_id, NULL); + + psArray *inputExps = NULL; + { + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddS32(where, PS_LIST_TAIL, "det_id", 0, "==", + (psS32)atoi(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return NULL; + } + inputExps = detInputExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + } + if (!inputExps) { + psError(PS_ERR_UNKNOWN, false, "no detInputExp rows found"); + return NULL; + } + + // find rawImfiles associated with detInputExps + psArray *rawImfiles = NULL; + { + psMetadata *where = psMetadataAlloc(); + for (long i = 0; i < psArrayLength(inputExps); i++) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", + PS_META_DUPLICATE_OK, "==", + ((detInputExpRow *)inputExps->data[i])->exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(inputExps); + psFree(where); + return NULL; + } + } + psFree(inputExps); + rawImfiles = rawImfileSelectRowObjects(config->dbh, where, 0); + // XXX this really should be sorted for uniqueness + psFree(where); + } + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found"); + return NULL; + } + + return rawImfiles; +} +#endif + +static bool addnormalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // make sure that there is a respondoing entry in detNormalizedStatImfile + // select * from detNormalizedStatImfile + // by det_id, iteration, class_id + // where det_id, iteration, class_id is not in detNormalizedImfile + psString query = psStringCopy( + "SELECT" + " detNormalizedStatImfile.*" + " FROM detNormalizedStatImfile" + " LEFT JOIN detNormalizedImfile" + " USING(det_id, iteration, class_id)" + " WHERE" + " detNormalizedImfile.det_id IS NULL" + " AND detNormalizedImfile.iteration IS NULL" + " AND detNormalizedImfile.class_id IS NULL" + ); + + { + // build a query to search by det_id, iteration, class_id + psMetadata *where = psMetadataAlloc(); + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + psFree(where); + psFree(query); + return false; + } + if (det_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "det_id", 0, "==", det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + psFree(query); + return false; + } + } + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + psFree(where); + psFree(query); + return false; + } + // always set iteration + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", iteration)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + psFree(query); + return false; + } + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + psFree(where); + psFree(query); + return false; + } + if (class_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "class_id", 0, "==", class_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class_id"); + psFree(where); + psFree(query); + return false; + } + } + + // there's not + psString whereClause = psDBGenerateWhereConditionSQL(where, "detNormalizedStatImfile"); + psFree(where); + if (whereClause) { + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + // convert metadata into a detNormalizedStatImfile object + detNormalizedStatImfileRow *statImfile = detNormalizedStatImfileObjectFromMetadata(row); + // convert detNormalizedStatImfile object into a detNormalizedImfile + detNormalizedImfileRow *normalizedImfile = detNormalizedStatToDetNormalizedmfile(config, statImfile); + psFree(statImfile); + if (!normalizedImfile) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to convert detStackedImfile to detNormalizedStatImfile"); + psFree(output); + return false; + } + // insert detNormlized Stat object into the database + if (!detNormalizedImfileInsertObject(config->dbh, normalizedImfile)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(normalizedImfile); + psFree(output); + } + psFree(normalizedImfile); + } + + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool normalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = psStringCopy( + "SELECT" + " detNormalizedImfile.*" + " FROM detNormalizedImfile" + " JOIN detRun" + " USING(det_id, iteration)" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + ); + + if (config->where) { + bool status; + int iteration = psMetadataLookupS32 (&status, config->where, "iteration"); + if (status) { + psMetadataRemoveKey (config->where, "iteration"); + psMetadataAddS32 (config->where, PS_LIST_TAIL, "iteration", 0, "==", iteration); + } + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detNormalizedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detNormalizedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detNormalizedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detNormalizedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertnormalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_revertnormalizedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detNormalizedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool tonormalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("dettool_tonormalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingNormExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addnormalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id, recip, -bg, -bg_stdev + // are required + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + return false; + } + if (!recipe) { + psError(PS_ERR_UNKNOWN, true, "-recip is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + + // optional + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + // iteration has a default value + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); return false; + } + + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + // optional + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + psString query = pxDataGet("dettool_tonormalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psStringAppend(&query, + "WHERE det_id = %s" + " AND iteration = %d", det_id, iteration); + + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + psFree(output); + + // create a new detProcessedImfile object + detNormalizedExpRow *detRow = detNormalizedExpRowAlloc( + (psS32)atoll(det_id), + iteration, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + code + ); + + // insert the new row into the detProcessedImfile table + if (!detNormalizedExpInsertObject(config->dbh, detRow)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detRow); + return false; + } + + psFree(detRow); + + return true; +} + + +static bool normalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = psStringCopy( + "SELECT" + " detNormalizedExp.*" + " FROM detNormalizedExp" + " JOIN detRun" + " USING(det_id, iteration)" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereSQL(config->where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detNormalizedExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detNormalizedExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detNormalizedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static detNormalizedImfileRow *detNormalizedStatToDetNormalizedmfile(pxConfig *config, detNormalizedStatImfileRow *statImfile) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(statImfile, NULL); + + bool status = false; + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + // optional + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + // optional + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + return detNormalizedImfileRowAlloc( + statImfile->det_id, + statImfile->iteration, + statImfile->class_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + code + ); +} + + +static bool revertnormalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_revertnormalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detNormalizedExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool toresidimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("dettool_toresidimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingResidImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addresidimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + + // defaults to 0 + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + return false; + } + + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + return false; + } + if (!class_id) { + psError(PS_ERR_UNKNOWN, false, "-class_id is required"); + return false; + } + + psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id"); + return false; + } + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "-exp_id is required"); + return false; + } + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + return false; + } + if (!recipe) { + psError(PS_ERR_UNKNOWN, true, "-recip is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + psF64 bg_skewness = psMetadataLookupF64(&status, config->args, "-bg_skewness"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_skewness"); + return false; + } + psF64 bg_kurtosis = psMetadataLookupF64(&status, config->args, "-bg_kurtosis"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_kurtosis"); + return false; + } + psF64 bin_stdev = psMetadataLookupF64(&status, config->args, "-bin_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bin_stdev"); + return false; + } + + // optional + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + // optional + psF64 fringe_0 = psMetadataLookupF64(&status, config->args, "-fringe_0"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_0"); + return false; + } + psF64 fringe_1 = psMetadataLookupF64(&status, config->args, "-fringe_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_1"); + return false; + } + psF64 fringe_2 = psMetadataLookupF64(&status, config->args, "-fringe_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_2"); + return false; + } + + psF64 fringe_resid_0 = psMetadataLookupF64(&status, config->args, "-fringe_resid_0"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_resid_0"); + return false; + } + psF64 fringe_resid_1 = psMetadataLookupF64(&status, config->args, "-fringe_resid_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_resid_1"); + return false; + } + psF64 fringe_resid_2 = psMetadataLookupF64(&status, config->args, "-fringe_resid_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_resid_2"); + return false; + } + + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + // create a new detResidImfileRow and insert it + if (!detResidImfileInsert( + config->dbh, + (psS64)atoll(det_id), + iteration, + (psS64)atoll(exp_id), + class_id, + uri, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + bg_skewness, + bg_kurtosis, + bin_stdev, + fringe_0, + fringe_1, + fringe_2, + fringe_resid_0, + fringe_resid_1, + fringe_resid_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool residimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + char *value = NULL; + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + // select detResidImfile.* + // by: + // where det_id, iteration, exp_id is not in detResidExp; + + psString query = psStringCopy( + "SELECT" + " detRun.det_type," + " detRun.mode," + " detResidImfile.*," + " rawExp.exp_time" + " FROM detResidImfile" + " JOIN detRun" + " USING(det_id, iteration)" + " JOIN rawExp" + " USING(exp_id)" + " WHERE" + ); + // NOTE the above WHERE is completed with the following line: + + // add the two required restrictions: detRun.state and detRun.mode + if ((value = psMetadataLookupStr(&status, config->args, "-select_state"))) { + psStringAppend(&query, " detRun.state = '%s'", value); + } else { + psStringAppend(&query, " detRun.state = 'run'"); + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detResidImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detResidImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detResidImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawResidImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertresidimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_revertresidimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detResidImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool toresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + /* +which returns a list of exposures for which all component class ids +have had residuals created. This list includes the detrend id, +iteration, exposure id, detrend type, and whether the exposure was +included in the stack for this iteration. + */ + + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("dettool_toresidexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingResidExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + /* +which returns a list of exposures for which all component class ids +have had residuals created. This list includes the detrend id, +iteration, exposure id, detrend type, and whether the exposure was +included in the stack for this iteration. + */ + + + // select detRun.det_id + // select detRun.iteration + // select detRun.det_type + // select detInputExp.exp_id + // select detInputExp.include + // by: + // find the current iteration bassed on det_id + // find all exp_ids in the current det_id/iteration from detInputExp + // compare to detInputExp.imfiles to derResidImfile by class_id + // and: + // detResidImfile.{det_id, iteration, exp_id} is not in detResidExp + + psString query = pxDataGet("dettool_toresidexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + { + // build a query to search by det_id, iteration, exp_id + psMetadata *where = psMetadataAlloc(); + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + if (!psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", (psS64)atoll(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + psFree(query); + return false; + } + + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + psFree(where); + psFree(query); + return false; + } + // always set iteration + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", iteration)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + psFree(query); + return false; + } + psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id"); + psFree(where); + psFree(query); + return false; + } + if (exp_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", 0, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + psFree(query); + return false; + } + } + + // there's not + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psFree(where); + if (whereClause) { + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no detResidImfile rows found"); + psFree(output); + return false; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + // convert metadata into a detResidExp object + detResidExpRow *residExp = mdToDetResidExp(config, row); + if (!residExp) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata to detResidExp"); + psFree(output); + return false; + } + // insert detResidExp object into the database + if (!detResidExpInsertObject(config->dbh, residExp)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(residExp); + psFree(output); + } + psFree(residExp); + } + + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static detResidExpRow *mdToDetResidExp(pxConfig *config, psMetadata *row) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(row, NULL); + + bool status = false; + // values from row + psS64 det_id = psMetadataLookupS64(&status, row, "det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for det_id"); + return false; + } + //if (isnan(det_id)) { + // psError(PS_ERR_UNKNOWN, true, "det_id is required"); + // return false; + //} + psS32 iteration = psMetadataLookupS32(&status, row, "iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for iteration"); + return false; + } + psS64 exp_id = psMetadataLookupS64(&status, row, "exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_id"); + return false; + } + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "exp_id is required"); + return false; + } + + // values from config + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + return false; + } + if (!recipe) { + psError(PS_ERR_UNKNOWN, true, "-recip is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + psF64 bg_skewness = psMetadataLookupF64(&status, config->args, "-bg_skewness"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_skewness"); + return false; + } + psF64 bg_kurtosis = psMetadataLookupF64(&status, config->args, "-bg_kurtosis"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_kurtosis"); + return false; + } + psF64 bin_stdev = psMetadataLookupF64(&status, config->args, "-bin_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bin_stdev"); + return false; + } + // optional + psF64 fringe_0 = psMetadataLookupF64(&status, config->args, "-fringe_0"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_0"); + return false; + } + psF64 fringe_1 = psMetadataLookupF64(&status, config->args, "-fringe_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_1"); + return false; + } + psF64 fringe_2 = psMetadataLookupF64(&status, config->args, "-fringe_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_2"); + return false; + } + + psF64 fringe_resid_0 = psMetadataLookupF64(&status, config->args, "-fringe_resid_0"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_resid_0"); + return false; + } + psF64 fringe_resid_1 = psMetadataLookupF64(&status, config->args, "-fringe_resid_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_resid_1"); + return false; + } + psF64 fringe_resid_2 = psMetadataLookupF64(&status, config->args, "-fringe_resid_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_resid_2"); + return false; + } + + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + // optional + bool reject = psMetadataLookupBool(&status, config->args, "-reject"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -reject"); + return false; + } + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + if (!path_base) { + psError(PS_ERR_UNKNOWN, true, "-path_base is required"); + return false; + } + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + // create a new detResidImfileRow and insert it + return detResidExpRowAlloc( + det_id, + iteration, + exp_id, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + bg_skewness, + bg_kurtosis, + bin_stdev, + fringe_0, + fringe_1, + fringe_2, + fringe_resid_0, + fringe_resid_1, + fringe_resid_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + !reject, + code + ); +} + +static bool residexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = psStringCopy( + "SELECT" + " detRun.mode," + " detResidExp.*," + " detInputExp.include" + " FROM detRun" + " JOIN detInputExp" + " USING(det_id, iteration)" + " JOIN detResidExp" + " USING(det_id, iteration, exp_id)" + " WHERE" + " detRun.state = 'run'" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detResidExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detResidExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detResidExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detResidExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_revertresidexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detResidExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool todetrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + /* which returns a list of detrend runs (with detrend id, iteration and + * detrend type) which have completed all residexps. + */ + + psString query = pxDataGet("dettool_todetrunsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detRejectExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool updateresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // build a query to search by det_id, iteration, exp_id + psMetadata *where = psMetadataAlloc(); + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + psFree(where); + return false; + } + if (det_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "det_id", 0, "==", det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return false; + } + } + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + psFree(where); + return false; + } + // always where iteration + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", iteration)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + return false; + } + psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id"); + psFree(where); + return false; + } + if (exp_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", 0, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + return false; + } + } + + // find the values we're going to set + // copy everything but det_id, iteration, & exp_id from the args and + // remove the '-' prefix + psMetadata *set = psMetadataAlloc(); + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + psFree(set); + psFree(where); + return false; + } + if (recipe) { + if (!psMetadataAddStr(set, PS_LIST_TAIL, "recipe", 0, "==", recipe)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item recipe"); + psFree(set); + psFree(where); + return false; + } + } + + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + psFree(set); + psFree(where); + return false; + } + if (!isnan(bg)) { + if (!psMetadataAddF64(set, PS_LIST_TAIL, "bg", 0, "==", bg)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item bg"); + psFree(set); + psFree(where); + return false; + } + } + + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + psFree(set); + psFree(where); + return false; + } + if (!isnan(bg_stdev)) { + if (!psMetadataAddF64(set, PS_LIST_TAIL, "bg_stdev", 0, "==", bg_stdev)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item bg_stdev"); + psFree(set); + psFree(where); + return false; + } + } + + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + psFree(set); + psFree(where); + return false; + } + if (!isnan(bg_mean_stdev)) { + if (!psMetadataAddF64(set, PS_LIST_TAIL, "bg_mean_stdev", 0, "==", bg_mean_stdev)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item bg_mean_stdev"); + psFree(set); + psFree(where); + return false; + } + } + + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + psFree(set); + psFree(where); + return false; + } + if (path_base) { + if (!psMetadataAddStr(set, PS_LIST_TAIL, "path_base", 0, "==", path_base)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item path_base"); + psFree(set); + psFree(where); + return false; + } + } + + bool reject = psMetadataLookupBool(&status, config->args, "-reject"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -reject"); + psFree(set); + psFree(where); + return false; + } + if (!psMetadataAddBool(set, PS_LIST_TAIL, "accept", 0, "==", !reject)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item accept"); + psFree(set); + psFree(where); + return false; + } + + long changed = psDBUpdateRows(config->dbh, "detResidExp", where, set); + psFree(set); + psFree(where); + + if (changed < 0) { + psError(PS_ERR_UNKNOWN, false, "no rows were updated"); + return false; + } + + return true; +} + +static bool adddetrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + + // optional + bool again = psMetadataLookupBool(&status, config->args, "-again"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -again"); + return false; + } + + psString query = pxDataGet("dettool_find_completed_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + { + // build a query to search by det_id, iteration, exp_id + psMetadata *where = psMetadataAlloc(); + bool status = false; + if (det_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", (psS64)atoll(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + psFree(query); + return false; + } + } + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + psFree(where); + psFree(query); + return false; + } + // always set iteration + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", iteration)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + psFree(query); + return false; + } + + // there's not + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psFree(where); + if (whereClause) { + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + // convert metadata into a detResidExp object + detRunSummaryRow *runSummary = mdToDetRunSummary(config, row); + if (!runSummary) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata to detResidExp"); + psFree(output); + return false; + } + // insert detResidExp object into the database + if (!detRunSummaryInsertObject(config->dbh, runSummary)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(runSummary); + psFree(output); + return false; + } + psFree(runSummary); + } + + psFree(output); + + // XXX this logic does not deal with the case of -code being set + if (again) { + if (!startNewIteration(config, (psS64)atoll(det_id))) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to start new iteration"); + return false; + } + } else { + // set detRun.state to stop + if (!setDetRunState(config, (psS64)atoll(det_id), "stop")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set detRun.state"); + return false; + } + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; +} + +static detRunSummaryRow *mdToDetRunSummary(pxConfig *config, psMetadata *row) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + // from row + psS64 det_id = psMetadataLookupS64(&status, row, "det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for det_id"); + return false; + } + psS32 iteration = psMetadataLookupS32(&status, row, "iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for iteration"); + return false; + } + + // from config->args + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return NULL; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return NULL; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return NULL; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return NULL; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return NULL; + } + // optional + bool accept = psMetadataLookupBool(&status, config->args, "-accept"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -accept"); + return NULL; + } + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + return detRunSummaryRowAlloc( + det_id, + iteration, + bg, + bg_stdev, + bg_mean_stdev, + accept, + code + ); +} + +static bool detrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = psStringCopy( + "SELECT DISTINCT\n" + " detRunSummary.*,\n" + " detRun.det_type,\n" + " detRun.mode\n" + " FROM detRun\n" + " JOIN detRunSummary\n" + " USING(det_id, iteration)\n" + " WHERE\n" + " detRun.state = 'run'\n" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detRunSummary.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detRunSummary.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawDetrendImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertdetrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_revertdetrunsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detRunSummary"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool updatedetrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id is required + // XXX this isn't strictly nessicary but not having the det_id complicates + // incrementing the iteration number + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + // either -rerun or -state must be specified + bool again = psMetadataLookupBool(&status, config->args, "-again"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -again"); + return false; + } + psString state = psMetadataLookupStr(&status, config->args, "-state"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -state"); + return false; + } + if (!(again || state)) { + psError(PS_ERR_UNKNOWN, true, "either -again or -state must be specified"); + return false; + } + if (again && state) { + psError(PS_ERR_UNKNOWN, true, "either -again or -state must be specified"); + return false; + } + + if (state) { + // set detRun.state to state + return setDetRunState(config, (psS64)atoll(det_id), state); + } + + // else + // -again + if (!startNewIteration(config, (psS64)atoll(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to start new iteration"); + return false; + } + + return true; +} + +static bool startNewIteration(pxConfig *config, psS64 det_id) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_start_new_iteration.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX this query was not restricted by det_id, resulting + // in an inconsistent UPDATE below. I added this AND clause + // though there may be a cleaner method (EAM 2006.10.08) + psStringAppend(&query, " WHERE det_id = %" PRId64 , det_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, false, "det_id %" PRId64 " not found", det_id); + psFree(output); + return false; + } + + // start a transaction so we don't end up with an incremented iteration + // count but no detInputExps + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // up the detRuns iteration count for for the single det_id we are + // operating on + // XXX this will have to changed in order to support multiple det_ids with + // a single invocation of this functions + psS32 newIteration = incrementIteration(config, det_id); + if (!newIteration) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + bool status = false; + psS64 exp_id = psMetadataLookupS64(&status, row, "exp_id"); + if (!status) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_id"); + psFree(output); + return false; + } + bool accept = psMetadataLookupBool(&status, row, "accept"); + if (!status) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for accept"); + psFree(output); + return false; + } + + // detResidExp.include is used to set detInputExp.include + if (!detInputExpInsert( + config->dbh, + det_id, + newIteration, + exp_id, + accept + ) + ) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + } + + psFree(output); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool rerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id is required + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + + // we have to support multipe exp_ids + psMetadataItem *item = psMetadataLookup(config->args, "-exp_id"); + if (!item) { + // this shouldn't actually happen when using psArgs + psError(PS_ERR_UNKNOWN, true, "-exp_id is required"); + return false; + } + + psList *exp_id_list = item->data.list; + psMetadata *where = psMetadataAlloc(); + // make sure that -exp_id was parsed correctly + // XXX this can be removed someday + if (item->type == PS_DATA_METADATA_MULTI) { + psListIterator *iter = psListIteratorAlloc(item->data.list, 0, false); + psMetadataItem *mItem = NULL; + while ((mItem = psListGetAndIncrement(iter))) { + psString exp_id = mItem->data.V; + // if exp_id is NULL then it means that -exp_id has not been + // specified + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, + "at least one -exp_id is required"); + psFree(where); + return false; + } + + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", + PS_META_DUPLICATE_OK, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(iter); + psFree(where); + return false; + } + } + psFree(iter); + } else { + psAbort("-exp_id was not parsed correctly (this should not happen"); + } + + // check that the specified exp_ids actually exist in the iteration zero + // detInputExp set + + // add the det_id & iteration == 0 to the where clause + if (!psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", + (psS64)atoll(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return false; + } + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", 0)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + return false; + } + + psArray *detrendExps = detInputExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + if (!detrendExps) { + psError(PS_ERR_UNKNOWN, false, "no rawExp rows found"); + psFree(where); + return false; + } + + // build a hash for the valid exp_ids + psHash *valid_exp_ids = psHashAlloc(psArrayLength(detrendExps)); + for (long i = 0; i < psArrayLength(detrendExps); i++) { + psString exp_idStr = psDBIntToString(((detInputExpRow *)detrendExps->data[i])->exp_id); + psHashAdd(valid_exp_ids, exp_idStr, detrendExps->data[i]); + psFree(exp_idStr); + } + psFree(detrendExps); + + // start a transaction so we don't end up with an incremented iteration + // count but no detInputExps + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(valid_exp_ids); + return false; + } + + // up the detRuns iteration count + psS32 newIteration = incrementIteration(config, (psS64)atoll(det_id)); + if (!newIteration) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(valid_exp_ids); + return false; + } + + // check exp_ids and build up an array of new detInputExp rows at the same + // time + psListIterator *iter = psListIteratorAlloc(exp_id_list, 0, false); + psMetadataItem *mItem = NULL; + psArray *newInputExps = psArrayAllocEmpty(psListLength(exp_id_list)); + while ((mItem = psListGetAndIncrement(iter))) { + detInputExpRow *inputExp = psHashLookup(valid_exp_ids, + (char *)mItem->data.V); + if (!inputExp) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + // invalid exp_id + psError(PS_ERR_UNKNOWN, false, "exp_id %s is invalid for det_id %s", + (char *)mItem->data.V, det_id); + psFree(iter); + psFree(valid_exp_ids); + return false; + } + detInputExpRow *newInputExp = detInputExpRowAlloc( + (psS64)atoll(det_id), + newIteration, + inputExp->exp_id, + true // use + ); + psArrayAdd(newInputExps, 0, newInputExp); + psFree(newInputExp); + } + psFree(iter); + psFree(valid_exp_ids); + + for (long i = 0; i < psArrayLength(newInputExps); i++) { + if (!detInputExpInsertObject(config->dbh, newInputExps->data[i])) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(newInputExps); + return false; + } + } + psFree(newInputExps); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool register_detrendMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + bool status = false; + psString det_type = psMetadataLookupStr(&status, config->args, "-det_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_type"); + return false; + } + if (!det_type) { + psError(PS_ERR_UNKNOWN, true, "-det_type is required"); + return false; + } + + psString filelevel = psMetadataLookupStr(&status, config->args, "-filelevel"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filelevel"); + return false; + } + if (!filelevel) { + psError(PS_ERR_UNKNOWN, true, "-filelevel is required"); + return false; + } + + psString workdir = psMetadataLookupStr(&status, config->args, "-workdir"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -workdir"); + return false; + } + + psString camera = psMetadataLookupStr(&status, config->args, "-inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst"); + return false; + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope"); + return false; + } + + psString exp_type = psMetadataLookupStr(&status, config->args, "-exp_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_type"); + return false; + } + + psString filter = psMetadataLookupStr(&status, config->args, "-filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter"); + return false; + } + + psF32 airmass_min = psMetadataLookupF32(&status, config->args, "-airmass_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass_min"); + return false; + } + + psF32 airmass_max = psMetadataLookupF32(&status, config->args, "-airmass_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass_max"); + return false; + } + + psF32 exp_time_min = psMetadataLookupF32(&status, config->args, "-exp_time_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time_min"); + return false; + } + + psF32 exp_time_max = psMetadataLookupF32(&status, config->args, "-exp_time_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time_max"); + return false; + } + + psF32 ccd_temp_min = psMetadataLookupF32(&status, config->args, "-ccd_temp_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp_min"); + return false; + } + + psF32 ccd_temp_max = psMetadataLookupF32(&status, config->args, "-ccd_temp_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp_max"); + return false; + } + + psF64 posang_min = psMetadataLookupF32(&status, config->args, "-posang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang_min"); + return false; + } + + psF64 posang_max = psMetadataLookupF32(&status, config->args, "-posang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang_max"); + return false; + } + + psF64 solang_min = psMetadataLookupF32(&status, config->args, "-solang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -solang_min"); + return false; + } + + psF64 solang_max = psMetadataLookupF32(&status, config->args, "-solang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -solang_max"); + return false; + } + + psTime *registered = NULL; + { + psString registeredStr = psMetadataLookupStr(&status, config->args, "-registered"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -registered"); + return false; + } + // pass through NULL as this is an optional field + if (registeredStr) { + registered = psTimeFromISO(registeredStr, PS_TIME_UTC); + } else { + registered = NULL; + } + } + + psTime *time_begin = NULL; + { + psString time_beginStr = psMetadataLookupStr(&status, config->args, "-time_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time_begin"); + return false; + } + // pass through NULL as this is an optional field + if (time_beginStr) { + time_begin = psTimeFromISO(time_beginStr, PS_TIME_UTC); + } else { + time_begin = NULL; + } + } + + psTime *time_end = NULL; + { + psString time_endStr = psMetadataLookupStr(&status, config->args, "-time_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time_end"); + return false; + } + // pass through NULL as this is an optional field + if (time_endStr) { + time_end = psTimeFromISO(time_endStr, PS_TIME_UTC); + } else { + time_end = NULL; + } + } + + psTime *use_begin = NULL; + { + psString use_beginStr = psMetadataLookupStr(&status, config->args, "-use_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -use_begin"); + return false; + } + // pass through NULL as this is an optional field + if (use_beginStr) { + use_begin = psTimeFromISO(use_beginStr, PS_TIME_UTC); + } else { + use_begin = NULL; + } + } + + psTime *use_end = NULL; + { + psString use_endStr = psMetadataLookupStr(&status, config->args, "-use_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -use_end"); + return false; + } + // pass through NULL as this is an optional field + if (use_endStr) { + use_end = psTimeFromISO(use_endStr, PS_TIME_UTC); + } else { + use_end = NULL; + } + } + + psString parent = psMetadataLookupStr(&status, config->args, "-parent"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -parent"); + return false; + } + + psString label = psMetadataLookupStr(&status, config->args, "-label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!detRunInsert(config->dbh, + 0, // det_id + 0, // the iteration is fixed at 0 + det_type, + "register", // mode + "register", // state + filelevel, + workdir, + camera, + telescope, + exp_type, + NULL, + filter, + airmass_min, + airmass_max, + exp_time_min, + exp_time_max, + ccd_temp_min, + ccd_temp_max, + posang_min, + posang_max, + registered, + time_begin, + time_end, + use_begin, + use_end, + solang_min, + solang_max, + label, // label + parent ? (psS64)atoll(parent) : 0 + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(registered); + return false; + } + psFree(registered); + + // print the new detRun + psS64 det_id = psDBLastInsertID(config->dbh); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id); + detRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check results + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 "(this should not happen)", det_id); + return false; + } + + if (!convertIdToStr(detRuns)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(detRuns); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, detRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detRuns); + return false; + } + psFree(detRuns); + + return true; +} + +static bool register_detrend_imfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + return false; + } + if (!class_id) { + psError(PS_ERR_UNKNOWN, true, "-class_id is required"); + return false; + } + + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + + // everything else is optional + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + if (!detRegisteredImfileInsert(config->dbh, + (psS64)atoll(det_id), + 0, // the iteration is fixed at 0 + class_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + 0 // fault code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static psS32 incrementIteration(pxConfig *config, psS64 det_id) +{ + // this function returns zero on error + PS_ASSERT_PTR_NON_NULL(config, 0); + + char *query = "UPDATE detRun SET iteration = iteration + 1 WHERE det_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to increment iteration for det_id %" PRId64, det_id); + return 0; + } + + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return 0; + } + + psArray *detRuns = detRunSelectRowObjects(config->dbh, where, 0); + psFree(where); + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "no detRun rows found"); + return 0; + } + + // sanity check the database + if (psArrayLength(detRuns) != 1) { + // this should no happen + psAbort( "database query return too many rows (this should not happen"); + } + + psS32 newIteration = ((detRunRow *)detRuns->data[0])->iteration; + psFree(detRuns); + + return newIteration; +} + +static bool setDetRunState(pxConfig *config, psS64 det_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "drop", 5) == 0) + || (strncmp(state, "register", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid detRun state: %s", state); + return false; + } + + char *query = "UPDATE detRun SET state = '%s' WHERE det_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64, det_id); + return false; + } + + return true; +} + +static bool isValidMode(pxConfig *config, const char *mode) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(mode, false); + + // check that state is a valid string value + if (!( + (strncmp(mode, "master", 7) == 0) + || (strncmp(mode, "verify", 7) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid detRun mode: %s", mode); + return false; + } + + return true; +} Index: trunk/ippTools/src/caltool.h =================================================================== --- trunk/ippTools/src/caltool.h (revision 15537) +++ trunk/ippTools/src/caltool.h (revision 15537) @@ -0,0 +1,6922 @@ +/* + * dettool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include + +#include "pxtools.h" +#include "dettool.h" + +static bool pendingMode(pxConfig *config); +static bool definebytagMode(pxConfig *config); +static bool definebyqueryMode(pxConfig *config); +static bool definebydetrunMode(pxConfig *config); +static bool runsMode(pxConfig *config); +static bool childlessrunMode(pxConfig *config); +static bool inputMode(pxConfig *config); +static bool rawMode(pxConfig *config); +// processedimfile +static bool toprocessedimfileMode(pxConfig *config); +static bool addprocessedimfileMode(pxConfig *config); +static bool processedimfileMode(pxConfig *config); +static bool revertprocessedimfileMode(pxConfig *config); +// processedexp +static bool toprocessedexpMode(pxConfig *config); +static bool addprocessedexpMode(pxConfig *config); +static bool processedexpMode(pxConfig *config); +static bool revertprocessedexpMode(pxConfig *config); +// stackedimfile +static bool tostackedMode(pxConfig *config); +static bool addstackedMode(pxConfig *config); +static bool stackedMode(pxConfig *config); +static bool revertstackedMode(pxConfig *config); +// normalizedstat +static bool tonormalizedstatMode(pxConfig *config); +static bool addnormalizedstatMode(pxConfig *config); +static bool normalizedstatMode(pxConfig *config); +static bool revertnormalizedstatMode(pxConfig *config); +// normalizedimfile +static bool tonormalizeMode(pxConfig *config); +static bool addnormalizedimfileMode(pxConfig *config); +static bool normalizedimfileMode(pxConfig *config); +static bool revertnormalizedimfileMode(pxConfig *config); +// normalizedexp +static bool tonormalizedexpMode(pxConfig *config); +static bool addnormalizedexpMode(pxConfig *config); +static bool normalizedexpMode(pxConfig *config); +static bool revertnormalizedexpMode(pxConfig *config); +// residimfile +static bool toresidimfileMode(pxConfig *config); +static bool addresidimfileMode(pxConfig *config); +static bool residimfileMode(pxConfig *config); +static bool revertresidimfileMode(pxConfig *config); +// residexp +static bool toresidexpMode(pxConfig *config); +static bool addresidexpMode(pxConfig *config); +static bool residexpMode(pxConfig *config); +static bool revertresidexpMode(pxConfig *config); +static bool updateresidexpMode(pxConfig *config); +// detrunsummary +static bool todetrunsummaryMode(pxConfig *config); +static bool adddetrunsummaryMode(pxConfig *config); +static bool detrunsummaryMode(pxConfig *config); +static bool revertdetrunsummaryMode(pxConfig *config); +static bool updatedetrunMode(pxConfig *config); +static bool rerunMode(pxConfig *config); +// register +static bool register_detrendMode(pxConfig *config); +static bool register_detrend_imfileMode(pxConfig *config); + +static detNormalizedStatImfileRow *detStackedToDetNormalizedStatImfile(pxConfig *config, detStackedImfileRow *stackedImfile); +static detNormalizedImfileRow *detNormalizedStatToDetNormalizedmfile(pxConfig *config, detNormalizedStatImfileRow *statImfile); +static detResidExpRow *mdToDetResidExp(pxConfig *config, psMetadata *row); +static detRunSummaryRow *mdToDetRunSummary(pxConfig *config, psMetadata *row); +//static psArray *validDetInputClassIds(pxConfig *config, const char *det_id); +//static psArray *searchInputImfiles(pxConfig *config, const char *det_id); +static detInputExpRow *rawDetrenTodetInputExpRow(rawExpRow *rawExp, psS64 det_id, psS32 iteration); +static psArray *searchRawImfiles(pxConfig *config, psMetadata *where); +static bool startNewIteration(pxConfig *config, psS64 det_id); +static psS32 incrementIteration(pxConfig *config, psS64 det_id); +static bool setDetRunState(pxConfig *config, psS64 det_id, const char *state); +static bool isValidMode(pxConfig *config, const char *mode); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = dettoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(DETTOOL_MODE_PENDING, pendingMode); + MODECASE(DETTOOL_MODE_DEFINEBYTAG, definebytagMode); + MODECASE(DETTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(DETTOOL_MODE_DEFINEBYDETRUN, definebydetrunMode); + MODECASE(DETTOOL_MODE_RUNS, runsMode); + MODECASE(DETTOOL_MODE_CHILDLESSRUN, childlessrunMode); + MODECASE(DETTOOL_MODE_INPUT, inputMode); + MODECASE(DETTOOL_MODE_RAW, rawMode); + // imfile + MODECASE(DETTOOL_MODE_TOPROCESSEDIMFILE,toprocessedimfileMode); + MODECASE(DETTOOL_MODE_ADDPROCESSEDIMFILE,addprocessedimfileMode); + MODECASE(DETTOOL_MODE_PROCESSEDIMFILE, processedimfileMode); + MODECASE(DETTOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileMode); + // exp + MODECASE(DETTOOL_MODE_TOPROCESSEDEXP, toprocessedexpMode); + MODECASE(DETTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpMode); + MODECASE(DETTOOL_MODE_PROCESSEDEXP, processedexpMode); + MODECASE(DETTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); + // stacked + MODECASE(DETTOOL_MODE_TOSTACKED, tostackedMode); + MODECASE(DETTOOL_MODE_ADDSTACKED, addstackedMode); + MODECASE(DETTOOL_MODE_STACKED, stackedMode); + MODECASE(DETTOOL_MODE_REVERTSTACKED, revertstackedMode); + // normalizedstat + MODECASE(DETTOOL_MODE_TONORMALIZEDSTAT, tonormalizedstatMode); + MODECASE(DETTOOL_MODE_ADDNORMALIZEDSTAT,addnormalizedstatMode); + MODECASE(DETTOOL_MODE_NORMALIZEDSTAT, normalizedstatMode); + MODECASE(DETTOOL_MODE_REVERTNORMALIZEDSTAT, revertnormalizedstatMode); + // normalizedimfile + MODECASE(DETTOOL_MODE_TONORMALIZE, tonormalizeMode); + MODECASE(DETTOOL_MODE_ADDNORMALIZEDIMFILE,addnormalizedimfileMode); + MODECASE(DETTOOL_MODE_NORMALIZEDIMFILE, normalizedimfileMode); + MODECASE(DETTOOL_MODE_REVERTNORMALIZEDIMFILE, revertnormalizedimfileMode); + // normalizedexp + MODECASE(DETTOOL_MODE_TONORMALIZEDEXP, tonormalizedexpMode); + MODECASE(DETTOOL_MODE_ADDNORMALIZEDEXP, addnormalizedexpMode); + MODECASE(DETTOOL_MODE_NORMALIZEDEXP, normalizedexpMode); + MODECASE(DETTOOL_MODE_REVERTNORMALIZEDEXP, revertnormalizedexpMode); + // residimfile + MODECASE(DETTOOL_MODE_TORESIDIMFILE, toresidimfileMode); + MODECASE(DETTOOL_MODE_ADDRESIDIMFILE, addresidimfileMode); + MODECASE(DETTOOL_MODE_RESIDIMFILE, residimfileMode); + MODECASE(DETTOOL_MODE_REVERTRESIDIMFILE,revertresidimfileMode); + // residexp + MODECASE(DETTOOL_MODE_TORESIDEXP, toresidexpMode); + MODECASE(DETTOOL_MODE_ADDRESIDEXP, addresidexpMode); + MODECASE(DETTOOL_MODE_RESIDEXP, residexpMode); + MODECASE(DETTOOL_MODE_REVERTRESIDEXP, revertresidexpMode); + MODECASE(DETTOOL_MODE_UPDATERESIDEXP, updateresidexpMode); + // detrunsummary + MODECASE(DETTOOL_MODE_TODETRUNSUMMARY, todetrunsummaryMode); + MODECASE(DETTOOL_MODE_ADDDETRUNSUMMARY, adddetrunsummaryMode); + MODECASE(DETTOOL_MODE_DETRUNSUMMARY, detrunsummaryMode); + MODECASE(DETTOOL_MODE_REVERTDETRUNSUMMARY, revertdetrunsummaryMode); + MODECASE(DETTOOL_MODE_UPDATEDETRUN, updatedetrunMode); + MODECASE(DETTOOL_MODE_RERUN, rerunMode); + // register + MODECASE(DETTOOL_MODE_REGISTER_DETREND, register_detrendMode); + MODECASE(DETTOOL_MODE_REGISTER_DETREND_IMFILE, register_detrend_imfileMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool pendingMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psString query = psStringCopy( + "SELECT" + " rawExp.*" + " FROM rawExp" + " LEFT JOIN detInputExp" + " ON rawExp.exp_id = detInputExp.exp_id" + " WHERE" + " detInputExp.exp_id IS NULL" + " AND rawExp.object != 'object'" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool definebytagMode(pxConfig *config) +{ + bool status = false; + + PS_ASSERT_PTR_NON_NULL(config, false); + + // what type of detRun is this? + // det_type is required + psString det_type = psMetadataLookupStr(&status, config->args, "-det_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_type"); + return false; + } + if (!det_type) { + psError(PS_ERR_UNKNOWN, true, "-det_type is required"); + return false; + } + + // optional + psString filelevel = psMetadataLookupStr(&status, config->args, "-filelevel"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filelevel"); + return false; + } + + psString workdir = psMetadataLookupStr(&status, config->args, "-workdir"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -workdir"); + return false; + } + if (!workdir) { + psError(PS_ERR_UNKNOWN, true, "-workdir is required"); + return false; + } + + // optional + psString mode = psMetadataLookupStr(&status, config->args, "-mode"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -mode"); + return false; + } + // check mode + if (mode && !isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalud mode"); + return false; + } + psString camera = psMetadataLookupStr(&status, config->args, "-inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst"); + return false; + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope"); + return false; + } + + psString exp_type = psMetadataLookupStr(&status, config->args, "-exp_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_type"); + return false; + } + + psString filter = psMetadataLookupStr(&status, config->args, "-filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter"); + return false; + } + + psF32 airmass_min = psMetadataLookupF32(&status, config->args, "-airmass_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass_min"); + return false; + } + + psF32 airmass_max = psMetadataLookupF32(&status, config->args, "-airmass_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass_max"); + return false; + } + + psF32 exp_time_min = psMetadataLookupF32(&status, config->args, "-exp_time_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time_min"); + return false; + } + + psF32 exp_time_max = psMetadataLookupF32(&status, config->args, "-exp_time_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time_max"); + return false; + } + + psF32 ccd_temp_min = psMetadataLookupF32(&status, config->args, "-ccd_temp_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp_min"); + return false; + } + + psF32 ccd_temp_max = psMetadataLookupF32(&status, config->args, "-ccd_temp_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp_max"); + return false; + } + + psF64 posang_min = psMetadataLookupF32(&status, config->args, "-posang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang_min"); + return false; + } + + psF64 posang_max = psMetadataLookupF32(&status, config->args, "-posang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang_max"); + return false; + } + psF64 solang_min = psMetadataLookupF32(&status, config->args, "-solang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -solang_min"); + return false; + } + + psF64 solang_max = psMetadataLookupF32(&status, config->args, "-solang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -solang_max"); + return false; + } + + psTime *registered = NULL; + { + psString registeredStr = psMetadataLookupStr(&status, config->args, "-registered"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -registered"); + return false; + } + // pass through NULL as this is an optional field + if (registeredStr) { + registered = psTimeFromISO(registeredStr, PS_TIME_UTC); + if (!registered) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", registeredStr); + return false; + } + } else { + registered = NULL; + } + } + + psTime *time_begin = NULL; + { + psString time_beginStr = psMetadataLookupStr(&status, config->args, "-time_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time_begin"); + return false; + } + // pass through NULL as this is an optional field + if (time_beginStr) { + time_begin = psTimeFromISO(time_beginStr, PS_TIME_UTC); + if (!time_begin) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", time_beginStr); + return false; + } + } else { + time_begin = NULL; + } + } + + psTime *time_end = NULL; + { + psString time_endStr = psMetadataLookupStr(&status, config->args, "-time_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time_end"); + return false; + } + // pass through NULL as this is an optional field + if (time_endStr) { + time_end = psTimeFromISO(time_endStr, PS_TIME_UTC); + if (!time_end) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", time_endStr); + return false; + } + } else { + time_end = NULL; + } + } + + psTime *use_begin = NULL; + { + psString use_beginStr = psMetadataLookupStr(&status, config->args, "-use_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -use_begin"); + return false; + } + // pass through NULL as this is an optional field + if (use_beginStr) { + use_begin = psTimeFromISO(use_beginStr, PS_TIME_UTC); + if (!use_begin) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", use_beginStr); + return false; + } + } else { + use_begin = NULL; + } + } + + psTime *use_end = NULL; + { + psString use_endStr = psMetadataLookupStr(&status, config->args, "-use_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -use_end"); + return false; + } + // pass through NULL as this is an optional field + if (use_endStr) { + use_end = psTimeFromISO(use_endStr, PS_TIME_UTC); + if (!use_end) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", use_endStr); + return false; + } + } else { + use_end = NULL; + } + } + + psString reduction = psMetadataLookupStr(&status, config->args, "-reduction"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -reduction"); + return false; + } + + psString label = psMetadataLookupStr(&status, config->args, "-label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label"); + return false; + } + + // we have to support multipe exp_ids + psMetadataItem *item = psMetadataLookup(config->args, "-exp_id"); + if (!item) { + // this shouldn't actually happen when using psArgs + psError(PS_ERR_UNKNOWN, true, "-exp_id is required"); + return false; + } + psMetadata *where = psMetadataAlloc(); + + // make sure that -exp_id was parsed correctly + // XXX this can be removed someday + if (item->type == PS_DATA_METADATA_MULTI) { + psListIterator *iter = psListIteratorAlloc(item->data.list, 0, false); + psMetadataItem *mItem = NULL; + while ((mItem = psListGetAndIncrement(iter))) { + psString exp_id = mItem->data.V; + // if exp_id is NULL then it means that -exp_id has not been + // specified + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, + "at least one -exp_id is required"); + psFree(where); + return false; + } + + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", + PS_META_DUPLICATE_OK, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(iter); + psFree(where); + return false; + } + } + psFree(iter); + } else { + psAbort( "-exp_id was not parsed correctly (this should not happen"); + } + + if (psListLength(where->list) < 1) { + psFree(where); + where = NULL; + } + + // check that the specified exp_ids actually exist + psArray *detrendExps = rawExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + if (!detrendExps) { + psError(PS_ERR_UNKNOWN, false, "no rawExp rows found"); + return false; + } + + // we should have one rawExp row per exp_id specified + if (psListLength(item->data.list) != psArrayLength(detrendExps)) { + psAbort( "an -exp_id matched more then one rawExp (this should not happen"); + + } + + // check to see if -filelevel was set on the command line + if (!filelevel) { + filelevel = psStringCopy(((rawExpRow *)detrendExps->data[0])->filelevel); + } + + // start a transaction so we don't end up with childlessed det_ids + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detrendExps); + return false; + } + + // the first iteration is always 0 + // XXX the camera name is set from the first inputExp + // XXX det_id + detRunInsert(config->dbh, + 0, + 0, + det_type, + mode, + "run", + filelevel, + workdir, + camera, + telescope, + exp_type, + reduction, + filter, + airmass_min, + airmass_max, + exp_time_min, + exp_time_max, + ccd_temp_min, + ccd_temp_max, + posang_min, + posang_max, + registered, + time_begin, + time_end, + use_begin, + use_end, + solang_min, + solang_max, + label, + 0 // parent + ); + psFree(registered); + psFree(time_begin); + psFree(time_end); + psFree(use_begin); + psFree(use_end); + psS64 det_id = psDBLastInsertID(config->dbh); + + // create new detInputExp row(s) from the rawExp row(s) + psArray *inputExps = psArrayAllocEmpty(psArrayLength(detrendExps)); + for (long i = 0; i < psArrayLength(detrendExps); i++) { + detInputExpRow *inputExp = rawDetrenTodetInputExpRow( + detrendExps->data[i], + det_id, + 0 // the first iteration is explicitly 0 + ); + psArrayAdd(inputExps, 0, inputExp); + psFree(inputExp); + } + + psFree(detrendExps); + + // insert detInputExp objects into the database + if (!detInputExpInsertObjects(config->dbh, inputExps)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(inputExps); + return false; + } + psFree(inputExps); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // print the new det_id + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id); + detRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check results + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 " (this should not happen)", det_id); + return false; + } + + if (!convertIdToStr(detRuns)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(detRuns); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, detRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detRuns); + return false; + } + psFree(detRuns); + + return true; +} + +#if 0 +// This function is used to convert the det_id from an int, as it is used +// internally, to be a string for external use. The rational being that we may +// want to change how det_id is generated in the future and don't want to +// external programs to become depending on this value being an int. +static bool convertDetIdToStr(psArray *mds) +{ + PS_ASSERT_PTR_NON_NULL(mds, false); + + for (long i = 0; i < psArrayLength(mds); i++) { + psMetadata *md = mds->data[i]; + bool status = false; + psS32 det_id = psMetadataLookupS32(&status, md, "det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for det_id"); + return false; + } + psMetadataRemoveKey(md, "det_id"); + psString det_idStr = psDBIntToString((psU64)det_id); + psMetadataAddStr(mds->data[i], PS_LIST_HEAD, "det_id", 0, NULL, det_idStr); + psFree(det_idStr); + } + + return true; +} +#endif + +static bool definebyqueryMode(pxConfig *config) +{ + bool status = false; + + PS_ASSERT_PTR_NON_NULL(config, false); + + // what type of detRun is this? + // det_type is required + psString det_type = psMetadataLookupStr(&status, config->args, "-det_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_type"); + return false; + } + if (!det_type) { + psError(PS_ERR_UNKNOWN, true, "-det_type is required"); + return false; + } + + // optional + psString filelevel = psMetadataLookupStr(&status, config->args, "-filelevel"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filelevel"); + return false; + } + + psString workdir = psMetadataLookupStr(&status, config->args, "-workdir"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -workdir"); + return false; + } + if (!workdir) { + psError(PS_ERR_UNKNOWN, true, "-workdir is required"); + return false; + } + psString camera = psMetadataLookupStr(&status, config->args, "-inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst"); + return false; + } + if (!camera) { + psError(PS_ERR_UNKNOWN, false, "-inst is required"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + { + bool status = false; + psString exp_type = psMetadataLookupStr(&status, config->args, "-select_exp_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_exp_type"); + return false; + } + if (exp_type) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_type", 0, "==", exp_type)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_type"); + psFree(where); + return false; + } + } + + // map -inst -> camera + psString camera = psMetadataLookupStr(&status, config->args, "-select_inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_inst"); + return false; + } + if (camera) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "camera", 0, "==", camera)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item inst"); + psFree(where); + return false; + } + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-select_telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_telescope"); + return false; + } + if (telescope) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "telescope", 0, "==", telescope)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item telescope"); + psFree(where); + return false; + } + } + + psString filter = psMetadataLookupStr(&status, config->args, "-select_filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_filter"); + return false; + } + if (filter) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "filter", 0, "==", filter)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item filter"); + psFree(where); + return false; + } + } + + psString uri = psMetadataLookupStr(&status, config->args, "-select_uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_uri"); + return false; + } + if (uri) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "uri", 0, "==", uri)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item uri"); + psFree(where); + return false; + } + } + + psString dateobs_begin = psMetadataLookupStr(&status, config->args, "-select_dateobs_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_dateobs_begin"); + psFree(where); + return false; + } + if (dateobs_begin) { + psTime *time = psTimeFromISO(dateobs_begin, PS_TIME_UTC); + if (!time) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", dateobs_begin); + psFree(time); + psFree(where); + return false; + } + if (!psMetadataAddTime(where, PS_LIST_TAIL, "dateobs", PS_META_DUPLICATE_OK, ">=", time)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item dateobs"); + psFree(time); + psFree(where); + return false; + } + psFree(time); + } + + psString dateobs_end = psMetadataLookupStr(&status, config->args, "-select_dateobs_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_dateobs_end"); + psFree(where); + return false; + } + if (dateobs_end) { + psTime *time = psTimeFromISO(dateobs_end, PS_TIME_UTC); + if (!time) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", dateobs_end); + psFree(time); + psFree(where); + return false; + } + if (!psMetadataAddTime(where, PS_LIST_TAIL, "dateobs", PS_META_DUPLICATE_OK, "<", time)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item dateobs"); + psFree(time); + psFree(where); + return false; + } + psFree(time); + } + + /** selection based on airmass range **/ + psF32 select_airmass_min = psMetadataLookupF32(&status, config->args, "-select_airmass_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_airmass_min"); + psFree(where); + return false; + } + if (isfinite(select_airmass_min)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "airmass", PS_META_DUPLICATE_OK, ">=", select_airmass_min)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item airmass"); + psFree(where); + return false; + } + } + psF32 select_airmass_max = psMetadataLookupF32(&status, config->args, "-select_airmass_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_airmass_max"); + psFree(where); + return false; + } + if (isfinite(select_airmass_max)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "airmass", PS_META_DUPLICATE_OK, "<=", select_airmass_max)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item airmass"); + psFree(where); + return false; + } + } + + psF32 select_sat_pixel_frac_max = psMetadataLookupF32(&status, config->args, "-select_sat_pixel_frac_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_sat_pixel_frac_max"); + psFree(where); + return false; + } + if (isfinite(select_sat_pixel_frac_max)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "sat_pixel_frac", PS_META_DUPLICATE_OK, "<=", select_sat_pixel_frac_max)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item sat_pixel_frac"); + psFree(where); + return false; + } + } + + /** selection based on exp_time range **/ + psF32 select_exp_time_min = psMetadataLookupF32(&status, config->args, "-select_exp_time_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_exp_time_min"); + psFree(where); + return false; + } + if (isfinite(select_exp_time_min)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "exp_time", PS_META_DUPLICATE_OK, ">=", select_exp_time_min)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_time"); + psFree(where); + return false; + } + } + psF32 select_exp_time_max = psMetadataLookupF32(&status, config->args, "-select_exp_time_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_exp_time_max"); + psFree(where); + return false; + } + if (isfinite(select_exp_time_max)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "exp_time", PS_META_DUPLICATE_OK, "<=", select_exp_time_max)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_time"); + psFree(where); + return false; + } + } + + /** selection based on ccd_temp range **/ + psF32 select_ccd_temp_min = psMetadataLookupF32(&status, config->args, "-select_ccd_temp_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_ccd_temp_min"); + psFree(where); + return false; + } + if (isfinite(select_ccd_temp_min)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "ccd_temp", PS_META_DUPLICATE_OK, ">=", select_ccd_temp_min)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item ccd_temp"); + psFree(where); + return false; + } + } + psF32 select_ccd_temp_max = psMetadataLookupF32(&status, config->args, "-select_ccd_temp_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_ccd_temp_max"); + psFree(where); + return false; + } + if (isfinite(select_ccd_temp_max)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "ccd_temp", PS_META_DUPLICATE_OK, "<=", select_ccd_temp_max)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item ccd_temp"); + psFree(where); + return false; + } + } + + /** selection based on posang **/ + psF32 select_posang_min = psMetadataLookupF32(&status, config->args, "-select_posang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_posang_min"); + psFree(where); + return false; + } + if (isfinite(select_posang_min)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "posang", PS_META_DUPLICATE_OK, ">=", select_posang_min)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item posang"); + psFree(where); + return false; + } + } + psF32 select_posang_max = psMetadataLookupF32(&status, config->args, "-select_posang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_posang_max"); + psFree(where); + return false; + } + if (isfinite(select_posang_max)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "posang", PS_META_DUPLICATE_OK, "<=", select_posang_max)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item posang"); + psFree(where); + return false; + } + } + + /** selection based on solang **/ + psF32 select_solang_min = psMetadataLookupF32(&status, config->args, "-select_solang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_solang_min"); + psFree(where); + return false; + } + if (isfinite(select_solang_min)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "solang", PS_META_DUPLICATE_OK, ">=", select_solang_min)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item solang"); + psFree(where); + return false; + } + } + psF32 select_solang_max = psMetadataLookupF32(&status, config->args, "-select_solang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_solang_max"); + psFree(where); + return false; + } + if (isfinite(select_solang_max)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "solang", PS_META_DUPLICATE_OK, "<=", select_solang_max)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item solang"); + psFree(where); + return false; + } + } + + } + + if (!psListLength(where->list)) { + psFree(where); + where = NULL; + } + + // there is some namespace overlap between the names of the fields we'd + // like to search by to setup a detrun and the names of the fields we'd + // like to assign values to so I've seperated them but prepending set- to + // the assigned values + + // optional + psString mode = psMetadataLookupStr(&status, config->args, "-mode"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -mode"); + return false; + } + // check mode + if (mode && !isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalud mode"); + return false; + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope"); + return false; + } + + psString filter = psMetadataLookupStr(&status, config->args, "-filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter"); + return false; + } + + psF32 airmass_min = psMetadataLookupF32(&status, config->args, "-airmass_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass_min"); + return false; + } + + psF32 airmass_max = psMetadataLookupF32(&status, config->args, "-airmass_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass_max"); + return false; + } + + psF32 exp_time_min = psMetadataLookupF32(&status, config->args, "-exp_time_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time_min"); + return false; + } + + psF32 exp_time_max = psMetadataLookupF32(&status, config->args, "-exp_time_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time_max"); + return false; + } + + psF32 ccd_temp_min = psMetadataLookupF32(&status, config->args, "-ccd_temp_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp_min"); + return false; + } + + psF32 ccd_temp_max = psMetadataLookupF32(&status, config->args, "-ccd_temp_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp_max"); + return false; + } + + psF64 posang_min = psMetadataLookupF32(&status, config->args, "-posang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang_min"); + return false; + } + + psF64 posang_max = psMetadataLookupF32(&status, config->args, "-posang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang_max"); + return false; + } + + psF64 solang_min = psMetadataLookupF32(&status, config->args, "-solang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -solang_min"); + return false; + } + + psF64 solang_max = psMetadataLookupF32(&status, config->args, "-solang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -solang_max"); + return false; + } + + psTime *registered = NULL; + { + psString registeredStr = psMetadataLookupStr(&status, config->args, "-registered"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -registered"); + return false; + } + // pass through NULL as this is an optional field + if (registeredStr) { + registered = psTimeFromISO(registeredStr, PS_TIME_UTC); + if (!registered) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", registeredStr); + return false; + } + } else { + registered = NULL; + } + } + + psTime *time_begin = NULL; + { + psString time_beginStr = psMetadataLookupStr(&status, config->args, "-time_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time_begin"); + return false; + } + // pass through NULL as this is an optional field + if (time_beginStr) { + time_begin = psTimeFromISO(time_beginStr, PS_TIME_UTC); + if (!time_begin) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", time_beginStr); + return false; + } + } else { + time_begin = NULL; + } + } + + psTime *time_end = NULL; + { + psString time_endStr = psMetadataLookupStr(&status, config->args, "-time_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time_end"); + return false; + } + // pass through NULL as this is an optional field + if (time_endStr) { + time_end = psTimeFromISO(time_endStr, PS_TIME_UTC); + if (!time_end) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", time_endStr); + return false; + } + } else { + time_end = NULL; + } + } + + psTime *use_begin = NULL; + { + psString use_beginStr = psMetadataLookupStr(&status, config->args, "-use_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -use_begin"); + return false; + } + // pass through NULL as this is an optional field + if (use_beginStr) { + use_begin = psTimeFromISO(use_beginStr, PS_TIME_UTC); + if (!use_begin) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", use_beginStr); + return false; + } + } else { + use_begin = NULL; + } + } + + psTime *use_end = NULL; + { + psString use_endStr = psMetadataLookupStr(&status, config->args, "-use_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -use_end"); + return false; + } + // pass through NULL as this is an optional field + if (use_endStr) { + use_end = psTimeFromISO(use_endStr, PS_TIME_UTC); + if (!use_end) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", use_endStr); + return false; + } + } else { + use_end = NULL; + } + } + + psString reduction = psMetadataLookupStr(&status, config->args, "-reduction"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -reduction"); + return false; + } + + psString label = psMetadataLookupStr(&status, config->args, "-label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label"); + return false; + } + + bool simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return false; + } + + bool pretend = psMetadataLookupBool(&status, config->args, "-pretend"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -pretend"); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return false; + } + + // search for rawExps with the specified options + psArray *detrendExps = rawExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + // make sure that we found at least one rawExp + if (!detrendExps) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return false; + } + if (!psArrayLength(detrendExps)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(detrendExps); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return true; + } + + // check to see if -filelevel was set on the command line + if (!filelevel) { + filelevel = psStringCopy(((rawExpRow *)detrendExps->data[0])->filelevel); + } + + if (pretend) { + // negative simple so the default is true + if (!rawExpPrintObjects(stdout, detrendExps, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detrendExps); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return false; + } + psFree(detrendExps); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return true; + } + + // start a transaction so we don't end up with childlessed det_ids + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detrendExps); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return false; + } + + // the first iteration is always 0 + // XXX det_id + detRunInsert(config->dbh, + 0, // det_id + 0, // iteration + det_type, + mode, + "run", // state + filelevel, + workdir, + camera, + telescope, + "NA", + reduction, + filter, + airmass_min, + airmass_max, + exp_time_min, + exp_time_max, + ccd_temp_min, + ccd_temp_max, + posang_min, + posang_max, + registered, + time_begin, + time_end, + use_begin, + use_end, + solang_min, + solang_max, + label, + 0 // parent + ); + psFree(registered); + psFree(time_begin); + psFree(time_end); + psFree(use_end); + psFree(use_begin); + psFree(use_end); + psS64 det_id = psDBLastInsertID(config->dbh); + + // create new detInputExp row(s) from the rawExp row(s) + psArray *inputExps = psArrayAllocEmpty(psArrayLength(detrendExps)); + for (long i = 0; i < psArrayLength(detrendExps); i++) { + detInputExpRow *inputExp = rawDetrenTodetInputExpRow( + detrendExps->data[i], + det_id, + 0 // the first iteration is explicitly 0 + ); + psArrayAdd(inputExps, 0, inputExp); + psFree(inputExp); + } + + psFree(detrendExps); + + // insert detInputExp objects into the database + if (!detInputExpInsertObjects(config->dbh, inputExps)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(inputExps); + return false; + } + psFree(inputExps); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + // print the new det_id + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id); + detRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check results + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 " (this should not happen)", det_id); + return false; + } + + // convert det_id to a string externaly + if (!convertIdToStr(detRuns)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert det_id to a string"); + psFree(detRuns); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, detRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detRuns); + return false; + } + psFree(detRuns); + + return true; +} + +static bool definebydetrunMode(pxConfig *config) +{ + bool status = false; + + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id is the only required input + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + + // lookup the detRun that we will be basing this one on + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", (psS64)atoll(det_id)); + detRuns = detRunSelectRowObjects(config->dbh, where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + // sanity check the result... we should have only found one det_id + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 " (this should not happen)", (psS64)atoll(det_id)); + return false; // unreachable + } + + // pull the detRun object out the result array + detRunRow *detRun = psMemIncrRefCounter(detRuns->data[0]); + + // discard the resultarray + psFree(detRuns); + + // set the det_id to 0/NULL so the database can assign it + detRun->det_id = 0; + + // reset the iteration to 0 + detRun->iteration = 0; + + // reset the state to "run" + psFree(detRun->state); + detRun->state = psStringCopy("run"); + + // walk through the optional values and update the detRun as required + psString det_type = psMetadataLookupStr(&status, config->args, "-set_det_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_det_type"); + return false; + } + if (det_type) { + psFree(detRun->det_type); + detRun->det_type = psStringCopy(det_type); + } + + psString mode = psMetadataLookupStr(&status, config->args, "-set_mode"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_mode"); + return false; + } + // check mode + if (mode && !isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalud mode"); + return false; + } + if (mode) { + psFree(detRun->mode); + detRun->mode = psStringCopy(mode); + } + + psString camera = psMetadataLookupStr(&status, config->args, "-set_inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_inst"); + return false; + } + if (camera) { + psFree(detRun->camera); + detRun->camera = psStringCopy(camera); + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-set_telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_telescope"); + return false; + } + if (telescope) { + psFree(detRun->telescope); + detRun->telescope = psStringCopy(telescope); + } + + psString exp_type = psMetadataLookupStr(&status, config->args, "-set_exp_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_exp_type"); + return false; + } + if (exp_type) { + psFree(detRun->exp_type); + detRun->exp_type = psStringCopy(exp_type); + } + + psString filelevel = psMetadataLookupStr(&status, config->args, "-set_filelevel"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_filelevel"); + return false; + } + if (filelevel) { + psFree(detRun->filelevel); + detRun->filelevel = psStringCopy(filelevel); + } + + psString workdir = psMetadataLookupStr(&status, config->args, "-set_workdir"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_workdir"); + return false; + } + if (workdir) { + psFree(detRun->workdir); + detRun->workdir = psStringCopy(workdir); + } + + psString filter = psMetadataLookupStr(&status, config->args, "-set_filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_filter"); + return false; + } + if (filter) { + psFree(detRun->filter); + detRun->filter = psStringCopy(filter); + } + + psF32 airmass_min = psMetadataLookupF32(&status, config->args, "-set_airmass_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_airmass_min"); + return false; + } + if (!isnan(airmass_min)) { + detRun->airmass_min = airmass_min; + } + + psF32 airmass_max = psMetadataLookupF32(&status, config->args, "-set_airmass_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_airmass_max"); + return false; + } + if (!isnan(airmass_max)) { + detRun->airmass_max = airmass_max; + } + + psF32 exp_time_min = psMetadataLookupF32(&status, config->args, "-set_exp_time_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_exp_time_min"); + return false; + } + if (!isnan(exp_time_min)) { + detRun->exp_time_min = exp_time_min; + } + + psF32 exp_time_max = psMetadataLookupF32(&status, config->args, "-set_exp_time_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_exp_time_max"); + return false; + } + if (!isnan(exp_time_max)) { + detRun->exp_time_max = exp_time_max; + } + + psF32 ccd_temp_min = psMetadataLookupF32(&status, config->args, "-set_ccd_temp_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_ccd_temp_min"); + return false; + } + if (!isnan(ccd_temp_min)) { + detRun->ccd_temp_min = ccd_temp_min; + } + + psF32 ccd_temp_max = psMetadataLookupF32(&status, config->args, "-set_ccd_temp_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_ccd_temp_max"); + return false; + } + if (!isnan(ccd_temp_max)) { + detRun->ccd_temp_max = ccd_temp_max; + } + + psF64 posang_min = psMetadataLookupF32(&status, config->args, "-set_posang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_posang_min"); + return false; + } + if (!isnan(posang_min)) { + detRun->posang_min = posang_min; + } + + psF64 posang_max = psMetadataLookupF32(&status, config->args, "-set_posang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_posang_max"); + return false; + } + if (!isnan(posang_max)) { + detRun->posang_max = posang_max; + } + + psF64 solang_min = psMetadataLookupF32(&status, config->args, "-set_solang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_solang_min"); + return false; + } + if (!isnan(solang_min)) { + detRun->solang_min = solang_min; + } + + psF64 solang_max = psMetadataLookupF32(&status, config->args, "-set_solang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_solang_max"); + return false; + } + if (!isnan(solang_max)) { + detRun->solang_max = solang_max; + } + + psString label = psMetadataLookupStr(&status, config->args, "-set_label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_label"); + return false; + } + if (label) { + detRun->label = label; + } + + psTime *registered = NULL; + { + psString registeredStr = psMetadataLookupStr(&status, config->args, "-set_registered"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_registered"); + return false; + } + // pass through NULL as this is an optional field + if (registeredStr) { + psFree(detRun->registered); + registered = psTimeFromISO(registeredStr, PS_TIME_UTC); + detRun->registered = psMemIncrRefCounter(registered); + psFree(registered); + } + } + + psTime *time_begin = NULL; + { + psString time_beginStr = psMetadataLookupStr(&status, config->args, "-set_time_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_time_begin"); + return false; + } + // pass through NULL as this is an optional field + if (time_beginStr) { + psFree(detRun->time_begin); + time_begin = psTimeFromISO(time_beginStr, PS_TIME_UTC); + detRun->time_begin = psMemIncrRefCounter(time_begin); + psFree(time_begin); + } + } + + psTime *time_end = NULL; + { + psString time_endStr = psMetadataLookupStr(&status, config->args, "-set_time_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_time_end"); + return false; + } + // pass through NULL as this is an optional field + if (time_endStr) { + psFree(detRun->time_end); + time_end = psTimeFromISO(time_endStr, PS_TIME_UTC); + detRun->time_end = psMemIncrRefCounter(time_end); + psFree(time_end); + } + } + + psTime *use_begin = NULL; + { + psString use_beginStr = psMetadataLookupStr(&status, config->args, "-set_use_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_use_begin"); + return false; + } + // pass through NULL as this is an optional field + if (use_beginStr) { + psFree(detRun->use_begin); + use_begin = psTimeFromISO(use_beginStr, PS_TIME_UTC); + detRun->use_begin = psMemIncrRefCounter(use_begin); + psFree(use_begin); + } + } + + psTime *use_end = NULL; + { + psString use_endStr = psMetadataLookupStr(&status, config->args, "-set_use_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_use_end"); + return false; + } + // pass through NULL as this is an optional field + if (use_endStr) { + psFree(detRun->use_end); + use_end = psTimeFromISO(use_endStr, PS_TIME_UTC); + detRun->use_end = psMemIncrRefCounter(use_end); + psFree(use_end); + } + } + + psString reduction = psMetadataLookupStr(&status, config->args, "-set_reduction"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_reduction"); + return false; + } + if (reduction) { + psFree(detRun->reduction); + detRun->reduction = psStringCopy(reduction); + } + + // create a metadata to restrict detInputExp's be in in the specified range + + psMetadata *time_filter = psMetadataAlloc(); + + { + psString timeStr = psMetadataLookupStr(&status, config->args, "-filter_input_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter_input_begin"); + return false; + } + // pass through NULL as this is an optional field + if (timeStr) { + psTime *time = psTimeFromISO(timeStr, PS_TIME_UTC); + if (!psMetadataAddTime(time_filter, PS_LIST_TAIL, "dateobs", PS_META_DUPLICATE_OK, ">=", time)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item dateobs"); + psFree(time); + psFree(time_filter); + return false; + } + psFree(time); + } + } + + { + psString timeStr = psMetadataLookupStr(&status, config->args, "-filter_input_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter_input_end"); + return false; + } + // pass through NULL as this is an optional field + if (timeStr) { + psTime *time = psTimeFromISO(timeStr, PS_TIME_UTC); + if (!psMetadataAddTime(time_filter, PS_LIST_TAIL, "dateobs", PS_META_DUPLICATE_OK, "<", time)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item dateobs"); + psFree(time); + psFree(time_filter); + return false; + } + psFree(time); + } + } + + + // start a transaction so we don't end up with childlessed det_ids + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(time_filter); + psFree(detRun); + return false; + } + + if (!detRunInsertObject(config->dbh, detRun)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(time_filter); + psFree(detRun); + return false; + } + psFree(detRun); + + // get the det_id + psS64 newDet_id = psDBLastInsertID(config->dbh); + + psString query = psStringCopy( + "INSERT INTO detInputExp" + " SELECT" + " %d," + " 0," + " detResidExp.exp_id," + " detResidExp.accept" + " FROM detResidExp" + " JOIN rawExp" + " USING(exp_id)" + " WHERE det_id = %d" + ); + + if (time_filter->list->n) { + psString whereClause = psDBGenerateWhereConditionSQL(time_filter, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(time_filter); + + if (!p_psDBRunQuery(config->dbh, query, (psS64)newDet_id, (psS64)atoll(det_id))) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // print the new det_id + psArray *newDetRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", newDet_id); + newDetRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!newDetRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check the result... we should have only found one det_id + if (psArrayLength(newDetRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 " (this should not happen)", newDet_id); + return false; // unreachable + } + + // convert det_id to a string externaly + if (!convertIdToStr(detRuns)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(detRuns); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, newDetRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(newDetRuns); + return false; + } + psFree(newDetRuns); + + return true; +} + +static bool runsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // XXX fix the hard coding of the table name + psArray *runs = psDBSelectRows(config->dbh, "detRun", config->where, 0); + if (!runs) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(runs)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(runs); + return true; + } + + // convert det_id to a string externaly + if (!convertIdToStr(runs)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(runs); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, runs, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(runs); + return false; + } + + psFree(runs); + + return true; +} + +static bool childlessrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = psStringCopy( + "SELECT DISTINCT\n" + " detRun.*\n" + " FROM detRun\n" + " LEFT JOIN detRun as foo\n" + " ON foo.parent = detRun.det_id\n" + " WHERE\n" + " detRun.state = 'stop'\n" + " AND detRun.mode = 'master'\n" + " AND foo.det_id IS NULL\n" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detRun"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // convert det_id to a string externaly + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static detInputExpRow *rawDetrenTodetInputExpRow(rawExpRow *rawExp, psS64 det_id, psS32 iteration) +{ + PS_ASSERT_PTR_NON_NULL(rawExp, NULL); + + return detInputExpRowAlloc( + det_id, + iteration, + rawExp->exp_id, + true // use + ); +} + +static bool inputMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // select detInputExp.* + // select rawExp.* + // by: + // exp_id + + psString query = psStringCopy( + "SELECT DISTINCT *" + " FROM detInputExp" + " JOIN rawExp" + " USING(exp_id)" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereSQL(config->where, "detInputExp"); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detInputExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool rawMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_raw.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool toprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("dettool_toprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingProcessedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static psArray *searchRawImfiles(pxConfig *config, psMetadata *where) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // use the default where if "where" is NULL + if (!where) { + where = config->where; + } + + // select exp_ids from detInputExp matching det_idp + // where query should be pre-generated + psArray *detInputExp = + detInputExpSelectRowObjects(config->dbh, where, 0); + if (!detInputExp) { + psError(PS_ERR_UNKNOWN, false, "no rawExp rows found"); + return NULL; + } + + // generate where query with just the exp_ids + psMetadata *where_exp_ids = psMetadataAlloc(); + for (long i = 0; i < psArrayLength(detInputExp); i++) { + detInputExpRow *row = detInputExp->data[i]; + if (!psMetadataAddS64(where_exp_ids, PS_LIST_TAIL, "exp_id", + PS_META_DUPLICATE_OK, "==", row->exp_id) + ) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(detInputExp); + psFree(where_exp_ids); + return NULL; + } + } + psFree(detInputExp); + + // select rawImfiles with matching exp_ids + psArray *rawImfiles = + rawImfileSelectRowObjects(config->dbh, where_exp_ids, 0); + psFree(where_exp_ids); + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found"); + return NULL; + } + + return rawImfiles; +} + +static bool addprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id, exp_id, class_id, uri, recipe, -bg, -bg_stdev + // are required + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id"); + return false; + } + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "-exp_id is required"); + return false; + } + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + return false; + } + if (!class_id) { + psError(PS_ERR_UNKNOWN, true, "-class_id is required"); + return false; + } + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + return false; + } + if (!recipe) { + psError(PS_ERR_UNKNOWN, true, "-recip is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + //psError(PS_ERR_UNKNOWN, true, "-bg is required"); + //return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + + // optional + psF64 fringe_0 = psMetadataLookupF64(&status, config->args, "-fringe_0"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_0"); + return false; + } + psF64 fringe_1 = psMetadataLookupF64(&status, config->args, "-fringe_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_1"); + return false; + } + psF64 fringe_2 = psMetadataLookupF64(&status, config->args, "-fringe_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_2"); + return false; + } + + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + // find the matching rawImfile by exp_id/class_id + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", 0, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + return false; + } + if (!psMetadataAddStr(where, PS_LIST_TAIL, "class_id", 0, "==", class_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class_id"); + psFree(where); + return false; + } + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + psArray *rawImfiles = rawImfileSelectRowObjects(config->dbh, where, 0); + psFree(where); + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found "); + return false; + } + + + // create a new detProcessedImfile object + detProcessedImfileRow *detRow = detProcessedImfileRowAlloc( + (psS64)atoll(det_id), + (psS64)atoll(exp_id), + class_id, + uri, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + fringe_0, + fringe_1, + fringe_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + code + ); + psFree(rawImfiles); + + // insert the new row into the detProcessedImfile table + if (!detProcessedImfileInsertObject(config->dbh, detRow)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detRow); + return false; + } + + psFree(detRow); + + return true; +} + +static bool toprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("dettool_toprocessedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id, exp_id, recip, -bg, -bg_stdev + // are required + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id"); + return false; + } + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "-exp_id is required"); + return false; + } + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + return false; + } + if (!recipe) { + psError(PS_ERR_UNKNOWN, true, "-recip is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + // optional + psF64 fringe_0 = psMetadataLookupF64(&status, config->args, "-fringe_0"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_0"); + return false; + } + psF64 fringe_1 = psMetadataLookupF64(&status, config->args, "-fringe_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_1"); + return false; + } + psF64 fringe_2 = psMetadataLookupF64(&status, config->args, "-fringe_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_2"); + return false; + } + + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + psString query = psStringCopy( + " SELECT DISTINCT" + " detProcessedImfile.det_id," + " detRun.iteration," + " detRun.det_type," + " detProcessedImfile.exp_id" + " FROM detRun" + " JOIN detInputExp" + " ON detRun.det_id = detInputExp.det_id" + " AND detRun.iteration = detInputExp.iteration" + " JOIN rawExp" + " ON detInputExp.exp_id = rawExp.exp_id" + " JOIN detProcessedImfile" + " ON detInputExp.det_id = detProcessedImfile.det_id" + " AND detInputExp.exp_id = detProcessedImfile.exp_id" + " LEFT JOIN detProcessedExp" + " ON detInputExp.det_id = detProcessedExp.det_id" + " AND detProcessedImfile.exp_id= detProcessedExp.exp_id" + " LEFT JOIN rawImfile" + " ON detInputExp.exp_id = rawImfile.exp_id" + " AND detProcessedImfile.class_id = rawImfile.class_id" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + " AND detProcessedExp.det_id IS NULL" + " AND detProcessedExp.exp_id IS NULL" + " AND detInputExp.include = 1" + " AND detRun.det_id = %s" + " AND detProcessedImfile.exp_id = '%s'" + " GROUP BY" + " detProcessedImfile.class_id," + " rawImfile.class_id," + " detRun.det_id" + " HAVING" + " COUNT(detProcessedImfile.class_id) = COUNT(rawImfile.class_id)" + ); + + if (!p_psDBRunQuery(config->dbh, query, det_id, exp_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + psFree(output); + + // create a new detProcessedImfile object + detProcessedExpRow *detRow = detProcessedExpRowAlloc( + (psS64)atoll(det_id), + (psS64)atoll(exp_id), + recipe, + bg, + bg_stdev, + bg_mean_stdev, + fringe_0, + fringe_1, + fringe_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + code + ); + + // insert the new row into the detProcessedImfile table + if (!detProcessedExpInsertObject(config->dbh, detRow)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detRow); + return false; + } + + psFree(detRow); + + return true; +} + +static bool processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = psStringCopy("SELECT * FROM detProcessedExp"); + + if (config->where) { + psString whereClause = psDBGenerateWhereSQL(config->where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detProcessedExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detProcessedExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_revertprocessedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detProcessedExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool tostackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("dettool_tostacked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingStackedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + char *value = NULL; + bool status = false; + + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = psStringCopy( + " SELECT DISTINCT" + " detRun.det_type," + " rawExp.exp_time," + " detProcessedImfile.*" + " FROM detProcessedImfile" + " JOIN detRun" + " USING(det_id)" + " JOIN detInputExp" + " ON detRun.det_id = detInputExp.det_id" + " AND detRun.iteration = detInputExp.iteration" + " AND detProcessedImfile.exp_id = detInputExp.exp_id" + " JOIN rawExp" + " ON rawExp.exp_id = detProcessedImfile.exp_id" + " WHERE" + ); + // NOTE the above WHERE is completed with the following line: + + // add the two required restrictions: detRun.state and detRun.mode + if ((value = psMetadataLookupStr(&status, config->args, "-select_state"))) { + psStringAppend(&query, " detRun.state = '%s'", value); + } else { + psStringAppend(&query, " detRun.state = 'run'"); + } + if ((value = psMetadataLookupStr(&status, config->args, "-select_mode"))) { + psStringAppend(&query, " AND detRun.mode = '%s'", value); + } else { + psStringAppend(&query, " AND detRun.mode = 'master'"); + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detProcessedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + { + bool status = false; + bool included = psMetadataLookupBool(&status, config->args, "-included"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + // restrict search to included imfiles + if (included) { + psStringAppend(&query, " AND detInputExp.include = 1"); + } + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detProcessedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detProcessedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_revertprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detProcessedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool addstackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id, class_id, uri, & recipe are required + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + return false; + } + if (!class_id) { + psError(PS_ERR_UNKNOWN, true, "-class_id is required"); + return false; + } + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + return false; + } + if (!recipe) { + psError(PS_ERR_UNKNOWN, true, "-recip is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + + // optional values + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + // default values + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + return false; + } + + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + // correlate the class_id against the input exposure(s) + + // we have to generate our own where clause as we want to search only by the + // det_id + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", + (psS64)atoll(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + return false; + } + + psArray *rawImfiles = searchRawImfiles(config, where); + psFree(where); + + bool valid_class_id = false; + if (rawImfiles) { + for (long i = 0; i < psArrayLength(rawImfiles); i++) { + if (strcmp(class_id, ((rawImfileRow *)rawImfiles->data[i])->class_id) == 0) { + valid_class_id = true; + break; + } + } + psFree(rawImfiles); + } + + if (!valid_class_id) { + psError(PS_ERR_UNKNOWN, true, + "class_id can not be correlated with the input exposures"); + return false; + } + + // create a new detStackedImfile object + detStackedImfileRow *stackedImfile = detStackedImfileRowAlloc( + (psS64)atoll(det_id), + iteration, + class_id, + uri, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + code + ); + + // insert the new row into the detProcessedImfile table + if (!detStackedImfileInsertObject(config->dbh, stackedImfile)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(stackedImfile); + return false; + } + + psFree(stackedImfile); + + return true; +} + +static bool stackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + // select detStackedImfile.* + // by: + // where det_id, iteration, class_id is not in detNormalizedImfile + + psString query = psStringCopy( + "SELECT" + " detStackedImfile.*" + " FROM detStackedImfile" + " JOIN detRun" + " USING(det_id, iteration)" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detStackedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detStackedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detStackedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertstackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_revertstacked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detStackedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +static bool tonormalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("dettool_tonormalizedstat.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingNormStatImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addnormalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // select * from detStackedImfile + // by det_id, iteration, class_id + // where det_id, iteration, class_id is not in detNormalizedStatImfile + psString query = psStringCopy( + "SELECT DISTINCT" + " detStackedImfile.*" + " FROM detStackedImfile" + " LEFT JOIN detNormalizedStatImfile" + " USING(det_id, iteration, class_id)" + " WHERE" + " detNormalizedStatImfile.det_id IS NULL" + " AND detNormalizedStatImfile.iteration IS NULL" + " AND detNormalizedStatImfile.class_id IS NULL" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detStackedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + // convert metadata into a detStackedImfile object + detStackedImfileRow *stackedImfile = detStackedImfileObjectFromMetadata(row); + // convert detStackedImfile object into a detNormalizedStat object + detNormalizedStatImfileRow *stat = detStackedToDetNormalizedStatImfile(config, stackedImfile); + psFree(stackedImfile); + if (!stat) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to convert detStackedImfile to detNormalizedStatImfile"); + psFree(output); + return false; + } + // insert detNormlized Stat object into the database + if (!detNormalizedStatImfileInsertObject(config->dbh, stat)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(stat); + psFree(output); + } + psFree(stat); + } + + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static detNormalizedStatImfileRow *detStackedToDetNormalizedStatImfile(pxConfig *config, detStackedImfileRow *stackedImfile) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(stackedImfile, NULL); + + bool status = false; + psF32 norm = psMetadataLookupF32(&status, config->args, "-norm"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -norm"); + return false; + } + //if (isnan(norm)) { + // psError(PS_ERR_UNKNOWN, true, "-norm is required"); + // return false; + //} + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + return detNormalizedStatImfileRowAlloc( + stackedImfile->det_id, + stackedImfile->iteration, + stackedImfile->class_id, + norm, + code + ); +} + + +static bool normalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = pxDataGet("dettool_normalizedstat.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "WHERE fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "WHERE fault = 0"); + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detNormalizedStatImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool revertnormalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_revertnormalizedstat.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detNormalizedStatImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +static bool tonormalizeMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + // select detNormalizedStatImfile.* + // by: + // where det_id, iteration, class_id is not in detNormalizedImfile + + psString query = psStringCopy( + "SELECT DISTINCT" + " detRun.det_type," + " detRun.workdir," + " rawExp.camera," + " detStackedImfile.uri," + " detNormalizedStatImfile.*" + " FROM detRun" + " JOIN detStackedImfile" + " USING(det_id, iteration)" + " JOIN detInputExp" + " USING(det_id, iteration)" + " JOIN rawExp" + " ON detInputExp.exp_id = rawExp.exp_id" + " JOIN detNormalizedStatImfile" + " ON detStackedImfile.det_id = detNormalizedStatImfile.det_id" + " AND detStackedImfile.iteration = detNormalizedStatImfile.iteration" + " AND detStackedImfile.class_id = detNormalizedStatImfile.class_id" + " LEFT JOIN detNormalizedImfile" + " ON detNormalizedStatImfile.det_id = detNormalizedImfile.det_id" + " AND detNormalizedStatImfile.iteration = detNormalizedImfile.iteration" + " AND detNormalizedStatImfile.class_id = detNormalizedImfile.class_id" + " WHERE" + " detNormalizedImfile.det_id IS NULL" + " AND detNormalizedImfile.iteration IS NULL" + " AND detNormalizedImfile.class_id IS NULL" + " AND detNormalizedStatImfile.fault = 0" + ); + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingNormImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +#if 0 +// XXX this function was left in commented as this method may be useful in the +// future +static psArray *validDetInputClassIds(pxConfig *config, const char *det_id) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + // det_id is input as a string because the fact that it is an integer + // is just a database impliementation detail. + PS_ASSERT_PTR_NON_NULL(det_id, NULL); + + psArray *rawImfiles = searchInputImfiles(config, det_id); + if (!rawImfiles) { + return NULL; + } + + psArray *valid_class_ids = NULL; + { + // All this jumping through hoops is so that we end up with unique + // values. PP thinks that making multiple passes through this array and + // deleting matched elements would end up being more expensive then a + // double sort and stagger scheme. JH thinks it would be cheaper to + // just do a unqiue sort and delete. So this is really just a cheap + // hack to avoid implimenting a unique sort function but at least it's + // stable. + psHash *hash = psHashAlloc(psArrayLength(rawImfiles)); + for (long i = 0; i < psArrayLength(rawImfiles); i++) { + psHashAdd(hash, + ((rawImfileRow *)rawImfiles->data[i])->class_id, + ((rawImfileRow *)rawImfiles->data[i])->class_id + ); + } + valid_class_ids = psHashToArray(hash); + psFree(hash); + } + psFree(rawImfiles); + + return valid_class_ids; +} + +static psArray *searchInputImfiles(pxConfig *config, const char *det_id) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + // det_id is input as a string because the fact that it is an integer + // is just a database impliementation detail. + PS_ASSERT_PTR_NON_NULL(det_id, NULL); + + psArray *inputExps = NULL; + { + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddS32(where, PS_LIST_TAIL, "det_id", 0, "==", + (psS32)atoi(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return NULL; + } + inputExps = detInputExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + } + if (!inputExps) { + psError(PS_ERR_UNKNOWN, false, "no detInputExp rows found"); + return NULL; + } + + // find rawImfiles associated with detInputExps + psArray *rawImfiles = NULL; + { + psMetadata *where = psMetadataAlloc(); + for (long i = 0; i < psArrayLength(inputExps); i++) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", + PS_META_DUPLICATE_OK, "==", + ((detInputExpRow *)inputExps->data[i])->exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(inputExps); + psFree(where); + return NULL; + } + } + psFree(inputExps); + rawImfiles = rawImfileSelectRowObjects(config->dbh, where, 0); + // XXX this really should be sorted for uniqueness + psFree(where); + } + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found"); + return NULL; + } + + return rawImfiles; +} +#endif + +static bool addnormalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // make sure that there is a respondoing entry in detNormalizedStatImfile + // select * from detNormalizedStatImfile + // by det_id, iteration, class_id + // where det_id, iteration, class_id is not in detNormalizedImfile + psString query = psStringCopy( + "SELECT" + " detNormalizedStatImfile.*" + " FROM detNormalizedStatImfile" + " LEFT JOIN detNormalizedImfile" + " USING(det_id, iteration, class_id)" + " WHERE" + " detNormalizedImfile.det_id IS NULL" + " AND detNormalizedImfile.iteration IS NULL" + " AND detNormalizedImfile.class_id IS NULL" + ); + + { + // build a query to search by det_id, iteration, class_id + psMetadata *where = psMetadataAlloc(); + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + psFree(where); + psFree(query); + return false; + } + if (det_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "det_id", 0, "==", det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + psFree(query); + return false; + } + } + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + psFree(where); + psFree(query); + return false; + } + // always set iteration + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", iteration)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + psFree(query); + return false; + } + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + psFree(where); + psFree(query); + return false; + } + if (class_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "class_id", 0, "==", class_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class_id"); + psFree(where); + psFree(query); + return false; + } + } + + // there's not + psString whereClause = psDBGenerateWhereConditionSQL(where, "detNormalizedStatImfile"); + psFree(where); + if (whereClause) { + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + // convert metadata into a detNormalizedStatImfile object + detNormalizedStatImfileRow *statImfile = detNormalizedStatImfileObjectFromMetadata(row); + // convert detNormalizedStatImfile object into a detNormalizedImfile + detNormalizedImfileRow *normalizedImfile = detNormalizedStatToDetNormalizedmfile(config, statImfile); + psFree(statImfile); + if (!normalizedImfile) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to convert detStackedImfile to detNormalizedStatImfile"); + psFree(output); + return false; + } + // insert detNormlized Stat object into the database + if (!detNormalizedImfileInsertObject(config->dbh, normalizedImfile)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(normalizedImfile); + psFree(output); + } + psFree(normalizedImfile); + } + + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool normalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = psStringCopy( + "SELECT" + " detNormalizedImfile.*" + " FROM detNormalizedImfile" + " JOIN detRun" + " USING(det_id, iteration)" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + ); + + if (config->where) { + bool status; + int iteration = psMetadataLookupS32 (&status, config->where, "iteration"); + if (status) { + psMetadataRemoveKey (config->where, "iteration"); + psMetadataAddS32 (config->where, PS_LIST_TAIL, "iteration", 0, "==", iteration); + } + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detNormalizedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detNormalizedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detNormalizedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detNormalizedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertnormalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_revertnormalizedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detNormalizedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool tonormalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("dettool_tonormalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingNormExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addnormalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id, recip, -bg, -bg_stdev + // are required + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + return false; + } + if (!recipe) { + psError(PS_ERR_UNKNOWN, true, "-recip is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + + // optional + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + // iteration has a default value + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); return false; + } + + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + // optional + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + psString query = pxDataGet("dettool_tonormalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psStringAppend(&query, + "WHERE det_id = %s" + " AND iteration = %d", det_id, iteration); + + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + psFree(output); + + // create a new detProcessedImfile object + detNormalizedExpRow *detRow = detNormalizedExpRowAlloc( + (psS32)atoll(det_id), + iteration, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + code + ); + + // insert the new row into the detProcessedImfile table + if (!detNormalizedExpInsertObject(config->dbh, detRow)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detRow); + return false; + } + + psFree(detRow); + + return true; +} + + +static bool normalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = psStringCopy( + "SELECT" + " detNormalizedExp.*" + " FROM detNormalizedExp" + " JOIN detRun" + " USING(det_id, iteration)" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereSQL(config->where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detNormalizedExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detNormalizedExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detNormalizedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static detNormalizedImfileRow *detNormalizedStatToDetNormalizedmfile(pxConfig *config, detNormalizedStatImfileRow *statImfile) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(statImfile, NULL); + + bool status = false; + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + // optional + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + // optional + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + return detNormalizedImfileRowAlloc( + statImfile->det_id, + statImfile->iteration, + statImfile->class_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + code + ); +} + + +static bool revertnormalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_revertnormalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detNormalizedExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool toresidimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("dettool_toresidimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingResidImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addresidimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + + // defaults to 0 + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + return false; + } + + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + return false; + } + if (!class_id) { + psError(PS_ERR_UNKNOWN, false, "-class_id is required"); + return false; + } + + psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id"); + return false; + } + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "-exp_id is required"); + return false; + } + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + return false; + } + if (!recipe) { + psError(PS_ERR_UNKNOWN, true, "-recip is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + psF64 bg_skewness = psMetadataLookupF64(&status, config->args, "-bg_skewness"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_skewness"); + return false; + } + psF64 bg_kurtosis = psMetadataLookupF64(&status, config->args, "-bg_kurtosis"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_kurtosis"); + return false; + } + psF64 bin_stdev = psMetadataLookupF64(&status, config->args, "-bin_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bin_stdev"); + return false; + } + + // optional + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + // optional + psF64 fringe_0 = psMetadataLookupF64(&status, config->args, "-fringe_0"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_0"); + return false; + } + psF64 fringe_1 = psMetadataLookupF64(&status, config->args, "-fringe_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_1"); + return false; + } + psF64 fringe_2 = psMetadataLookupF64(&status, config->args, "-fringe_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_2"); + return false; + } + + psF64 fringe_resid_0 = psMetadataLookupF64(&status, config->args, "-fringe_resid_0"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_resid_0"); + return false; + } + psF64 fringe_resid_1 = psMetadataLookupF64(&status, config->args, "-fringe_resid_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_resid_1"); + return false; + } + psF64 fringe_resid_2 = psMetadataLookupF64(&status, config->args, "-fringe_resid_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_resid_2"); + return false; + } + + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + // create a new detResidImfileRow and insert it + if (!detResidImfileInsert( + config->dbh, + (psS64)atoll(det_id), + iteration, + (psS64)atoll(exp_id), + class_id, + uri, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + bg_skewness, + bg_kurtosis, + bin_stdev, + fringe_0, + fringe_1, + fringe_2, + fringe_resid_0, + fringe_resid_1, + fringe_resid_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool residimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + char *value = NULL; + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + // select detResidImfile.* + // by: + // where det_id, iteration, exp_id is not in detResidExp; + + psString query = psStringCopy( + "SELECT" + " detRun.det_type," + " detRun.mode," + " detResidImfile.*," + " rawExp.exp_time" + " FROM detResidImfile" + " JOIN detRun" + " USING(det_id, iteration)" + " JOIN rawExp" + " USING(exp_id)" + " WHERE" + ); + // NOTE the above WHERE is completed with the following line: + + // add the two required restrictions: detRun.state and detRun.mode + if ((value = psMetadataLookupStr(&status, config->args, "-select_state"))) { + psStringAppend(&query, " detRun.state = '%s'", value); + } else { + psStringAppend(&query, " detRun.state = 'run'"); + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detResidImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detResidImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detResidImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawResidImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertresidimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_revertresidimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detResidImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool toresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + /* +which returns a list of exposures for which all component class ids +have had residuals created. This list includes the detrend id, +iteration, exposure id, detrend type, and whether the exposure was +included in the stack for this iteration. + */ + + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("dettool_toresidexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingResidExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + /* +which returns a list of exposures for which all component class ids +have had residuals created. This list includes the detrend id, +iteration, exposure id, detrend type, and whether the exposure was +included in the stack for this iteration. + */ + + + // select detRun.det_id + // select detRun.iteration + // select detRun.det_type + // select detInputExp.exp_id + // select detInputExp.include + // by: + // find the current iteration bassed on det_id + // find all exp_ids in the current det_id/iteration from detInputExp + // compare to detInputExp.imfiles to derResidImfile by class_id + // and: + // detResidImfile.{det_id, iteration, exp_id} is not in detResidExp + + psString query = pxDataGet("dettool_toresidexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + { + // build a query to search by det_id, iteration, exp_id + psMetadata *where = psMetadataAlloc(); + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + if (!psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", (psS64)atoll(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + psFree(query); + return false; + } + + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + psFree(where); + psFree(query); + return false; + } + // always set iteration + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", iteration)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + psFree(query); + return false; + } + psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id"); + psFree(where); + psFree(query); + return false; + } + if (exp_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", 0, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + psFree(query); + return false; + } + } + + // there's not + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psFree(where); + if (whereClause) { + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no detResidImfile rows found"); + psFree(output); + return false; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + // convert metadata into a detResidExp object + detResidExpRow *residExp = mdToDetResidExp(config, row); + if (!residExp) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata to detResidExp"); + psFree(output); + return false; + } + // insert detResidExp object into the database + if (!detResidExpInsertObject(config->dbh, residExp)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(residExp); + psFree(output); + } + psFree(residExp); + } + + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static detResidExpRow *mdToDetResidExp(pxConfig *config, psMetadata *row) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(row, NULL); + + bool status = false; + // values from row + psS64 det_id = psMetadataLookupS64(&status, row, "det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for det_id"); + return false; + } + //if (isnan(det_id)) { + // psError(PS_ERR_UNKNOWN, true, "det_id is required"); + // return false; + //} + psS32 iteration = psMetadataLookupS32(&status, row, "iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for iteration"); + return false; + } + psS64 exp_id = psMetadataLookupS64(&status, row, "exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_id"); + return false; + } + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "exp_id is required"); + return false; + } + + // values from config + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + return false; + } + if (!recipe) { + psError(PS_ERR_UNKNOWN, true, "-recip is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + psF64 bg_skewness = psMetadataLookupF64(&status, config->args, "-bg_skewness"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_skewness"); + return false; + } + psF64 bg_kurtosis = psMetadataLookupF64(&status, config->args, "-bg_kurtosis"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_kurtosis"); + return false; + } + psF64 bin_stdev = psMetadataLookupF64(&status, config->args, "-bin_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bin_stdev"); + return false; + } + // optional + psF64 fringe_0 = psMetadataLookupF64(&status, config->args, "-fringe_0"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_0"); + return false; + } + psF64 fringe_1 = psMetadataLookupF64(&status, config->args, "-fringe_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_1"); + return false; + } + psF64 fringe_2 = psMetadataLookupF64(&status, config->args, "-fringe_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_2"); + return false; + } + + psF64 fringe_resid_0 = psMetadataLookupF64(&status, config->args, "-fringe_resid_0"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_resid_0"); + return false; + } + psF64 fringe_resid_1 = psMetadataLookupF64(&status, config->args, "-fringe_resid_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_resid_1"); + return false; + } + psF64 fringe_resid_2 = psMetadataLookupF64(&status, config->args, "-fringe_resid_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_resid_2"); + return false; + } + + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + // optional + bool reject = psMetadataLookupBool(&status, config->args, "-reject"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -reject"); + return false; + } + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + if (!path_base) { + psError(PS_ERR_UNKNOWN, true, "-path_base is required"); + return false; + } + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + // create a new detResidImfileRow and insert it + return detResidExpRowAlloc( + det_id, + iteration, + exp_id, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + bg_skewness, + bg_kurtosis, + bin_stdev, + fringe_0, + fringe_1, + fringe_2, + fringe_resid_0, + fringe_resid_1, + fringe_resid_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + !reject, + code + ); +} + +static bool residexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = psStringCopy( + "SELECT" + " detRun.mode," + " detResidExp.*," + " detInputExp.include" + " FROM detRun" + " JOIN detInputExp" + " USING(det_id, iteration)" + " JOIN detResidExp" + " USING(det_id, iteration, exp_id)" + " WHERE" + " detRun.state = 'run'" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detResidExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detResidExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detResidExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detResidExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_revertresidexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detResidExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool todetrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + /* which returns a list of detrend runs (with detrend id, iteration and + * detrend type) which have completed all residexps. + */ + + psString query = pxDataGet("dettool_todetrunsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detRejectExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool updateresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // build a query to search by det_id, iteration, exp_id + psMetadata *where = psMetadataAlloc(); + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + psFree(where); + return false; + } + if (det_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "det_id", 0, "==", det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return false; + } + } + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + psFree(where); + return false; + } + // always where iteration + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", iteration)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + return false; + } + psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id"); + psFree(where); + return false; + } + if (exp_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", 0, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + return false; + } + } + + // find the values we're going to set + // copy everything but det_id, iteration, & exp_id from the args and + // remove the '-' prefix + psMetadata *set = psMetadataAlloc(); + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + psFree(set); + psFree(where); + return false; + } + if (recipe) { + if (!psMetadataAddStr(set, PS_LIST_TAIL, "recipe", 0, "==", recipe)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item recipe"); + psFree(set); + psFree(where); + return false; + } + } + + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + psFree(set); + psFree(where); + return false; + } + if (!isnan(bg)) { + if (!psMetadataAddF64(set, PS_LIST_TAIL, "bg", 0, "==", bg)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item bg"); + psFree(set); + psFree(where); + return false; + } + } + + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + psFree(set); + psFree(where); + return false; + } + if (!isnan(bg_stdev)) { + if (!psMetadataAddF64(set, PS_LIST_TAIL, "bg_stdev", 0, "==", bg_stdev)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item bg_stdev"); + psFree(set); + psFree(where); + return false; + } + } + + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + psFree(set); + psFree(where); + return false; + } + if (!isnan(bg_mean_stdev)) { + if (!psMetadataAddF64(set, PS_LIST_TAIL, "bg_mean_stdev", 0, "==", bg_mean_stdev)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item bg_mean_stdev"); + psFree(set); + psFree(where); + return false; + } + } + + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + psFree(set); + psFree(where); + return false; + } + if (path_base) { + if (!psMetadataAddStr(set, PS_LIST_TAIL, "path_base", 0, "==", path_base)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item path_base"); + psFree(set); + psFree(where); + return false; + } + } + + bool reject = psMetadataLookupBool(&status, config->args, "-reject"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -reject"); + psFree(set); + psFree(where); + return false; + } + if (!psMetadataAddBool(set, PS_LIST_TAIL, "accept", 0, "==", !reject)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item accept"); + psFree(set); + psFree(where); + return false; + } + + long changed = psDBUpdateRows(config->dbh, "detResidExp", where, set); + psFree(set); + psFree(where); + + if (changed < 0) { + psError(PS_ERR_UNKNOWN, false, "no rows were updated"); + return false; + } + + return true; +} + +static bool adddetrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + + // optional + bool again = psMetadataLookupBool(&status, config->args, "-again"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -again"); + return false; + } + + psString query = pxDataGet("dettool_find_completed_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + { + // build a query to search by det_id, iteration, exp_id + psMetadata *where = psMetadataAlloc(); + bool status = false; + if (det_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", (psS64)atoll(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + psFree(query); + return false; + } + } + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + psFree(where); + psFree(query); + return false; + } + // always set iteration + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", iteration)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + psFree(query); + return false; + } + + // there's not + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psFree(where); + if (whereClause) { + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + // convert metadata into a detResidExp object + detRunSummaryRow *runSummary = mdToDetRunSummary(config, row); + if (!runSummary) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata to detResidExp"); + psFree(output); + return false; + } + // insert detResidExp object into the database + if (!detRunSummaryInsertObject(config->dbh, runSummary)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(runSummary); + psFree(output); + return false; + } + psFree(runSummary); + } + + psFree(output); + + // XXX this logic does not deal with the case of -code being set + if (again) { + if (!startNewIteration(config, (psS64)atoll(det_id))) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to start new iteration"); + return false; + } + } else { + // set detRun.state to stop + if (!setDetRunState(config, (psS64)atoll(det_id), "stop")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set detRun.state"); + return false; + } + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; +} + +static detRunSummaryRow *mdToDetRunSummary(pxConfig *config, psMetadata *row) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + // from row + psS64 det_id = psMetadataLookupS64(&status, row, "det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for det_id"); + return false; + } + psS32 iteration = psMetadataLookupS32(&status, row, "iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for iteration"); + return false; + } + + // from config->args + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return NULL; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return NULL; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return NULL; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return NULL; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return NULL; + } + // optional + bool accept = psMetadataLookupBool(&status, config->args, "-accept"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -accept"); + return NULL; + } + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + return detRunSummaryRowAlloc( + det_id, + iteration, + bg, + bg_stdev, + bg_mean_stdev, + accept, + code + ); +} + +static bool detrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = psStringCopy( + "SELECT DISTINCT\n" + " detRunSummary.*,\n" + " detRun.det_type,\n" + " detRun.mode\n" + " FROM detRun\n" + " JOIN detRunSummary\n" + " USING(det_id, iteration)\n" + " WHERE\n" + " detRun.state = 'run'\n" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detRunSummary.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detRunSummary.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawDetrendImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertdetrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_revertdetrunsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detRunSummary"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool updatedetrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id is required + // XXX this isn't strictly nessicary but not having the det_id complicates + // incrementing the iteration number + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + // either -rerun or -state must be specified + bool again = psMetadataLookupBool(&status, config->args, "-again"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -again"); + return false; + } + psString state = psMetadataLookupStr(&status, config->args, "-state"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -state"); + return false; + } + if (!(again || state)) { + psError(PS_ERR_UNKNOWN, true, "either -again or -state must be specified"); + return false; + } + if (again && state) { + psError(PS_ERR_UNKNOWN, true, "either -again or -state must be specified"); + return false; + } + + if (state) { + // set detRun.state to state + return setDetRunState(config, (psS64)atoll(det_id), state); + } + + // else + // -again + if (!startNewIteration(config, (psS64)atoll(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to start new iteration"); + return false; + } + + return true; +} + +static bool startNewIteration(pxConfig *config, psS64 det_id) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_start_new_iteration.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX this query was not restricted by det_id, resulting + // in an inconsistent UPDATE below. I added this AND clause + // though there may be a cleaner method (EAM 2006.10.08) + psStringAppend(&query, " WHERE det_id = %" PRId64 , det_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, false, "det_id %" PRId64 " not found", det_id); + psFree(output); + return false; + } + + // start a transaction so we don't end up with an incremented iteration + // count but no detInputExps + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // up the detRuns iteration count for for the single det_id we are + // operating on + // XXX this will have to changed in order to support multiple det_ids with + // a single invocation of this functions + psS32 newIteration = incrementIteration(config, det_id); + if (!newIteration) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + bool status = false; + psS64 exp_id = psMetadataLookupS64(&status, row, "exp_id"); + if (!status) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_id"); + psFree(output); + return false; + } + bool accept = psMetadataLookupBool(&status, row, "accept"); + if (!status) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for accept"); + psFree(output); + return false; + } + + // detResidExp.include is used to set detInputExp.include + if (!detInputExpInsert( + config->dbh, + det_id, + newIteration, + exp_id, + accept + ) + ) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + } + + psFree(output); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool rerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id is required + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + + // we have to support multipe exp_ids + psMetadataItem *item = psMetadataLookup(config->args, "-exp_id"); + if (!item) { + // this shouldn't actually happen when using psArgs + psError(PS_ERR_UNKNOWN, true, "-exp_id is required"); + return false; + } + + psList *exp_id_list = item->data.list; + psMetadata *where = psMetadataAlloc(); + // make sure that -exp_id was parsed correctly + // XXX this can be removed someday + if (item->type == PS_DATA_METADATA_MULTI) { + psListIterator *iter = psListIteratorAlloc(item->data.list, 0, false); + psMetadataItem *mItem = NULL; + while ((mItem = psListGetAndIncrement(iter))) { + psString exp_id = mItem->data.V; + // if exp_id is NULL then it means that -exp_id has not been + // specified + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, + "at least one -exp_id is required"); + psFree(where); + return false; + } + + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", + PS_META_DUPLICATE_OK, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(iter); + psFree(where); + return false; + } + } + psFree(iter); + } else { + psAbort("-exp_id was not parsed correctly (this should not happen"); + } + + // check that the specified exp_ids actually exist in the iteration zero + // detInputExp set + + // add the det_id & iteration == 0 to the where clause + if (!psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", + (psS64)atoll(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return false; + } + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", 0)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + return false; + } + + psArray *detrendExps = detInputExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + if (!detrendExps) { + psError(PS_ERR_UNKNOWN, false, "no rawExp rows found"); + psFree(where); + return false; + } + + // build a hash for the valid exp_ids + psHash *valid_exp_ids = psHashAlloc(psArrayLength(detrendExps)); + for (long i = 0; i < psArrayLength(detrendExps); i++) { + psString exp_idStr = psDBIntToString(((detInputExpRow *)detrendExps->data[i])->exp_id); + psHashAdd(valid_exp_ids, exp_idStr, detrendExps->data[i]); + psFree(exp_idStr); + } + psFree(detrendExps); + + // start a transaction so we don't end up with an incremented iteration + // count but no detInputExps + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(valid_exp_ids); + return false; + } + + // up the detRuns iteration count + psS32 newIteration = incrementIteration(config, (psS64)atoll(det_id)); + if (!newIteration) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(valid_exp_ids); + return false; + } + + // check exp_ids and build up an array of new detInputExp rows at the same + // time + psListIterator *iter = psListIteratorAlloc(exp_id_list, 0, false); + psMetadataItem *mItem = NULL; + psArray *newInputExps = psArrayAllocEmpty(psListLength(exp_id_list)); + while ((mItem = psListGetAndIncrement(iter))) { + detInputExpRow *inputExp = psHashLookup(valid_exp_ids, + (char *)mItem->data.V); + if (!inputExp) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + // invalid exp_id + psError(PS_ERR_UNKNOWN, false, "exp_id %s is invalid for det_id %s", + (char *)mItem->data.V, det_id); + psFree(iter); + psFree(valid_exp_ids); + return false; + } + detInputExpRow *newInputExp = detInputExpRowAlloc( + (psS64)atoll(det_id), + newIteration, + inputExp->exp_id, + true // use + ); + psArrayAdd(newInputExps, 0, newInputExp); + psFree(newInputExp); + } + psFree(iter); + psFree(valid_exp_ids); + + for (long i = 0; i < psArrayLength(newInputExps); i++) { + if (!detInputExpInsertObject(config->dbh, newInputExps->data[i])) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(newInputExps); + return false; + } + } + psFree(newInputExps); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool register_detrendMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + bool status = false; + psString det_type = psMetadataLookupStr(&status, config->args, "-det_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_type"); + return false; + } + if (!det_type) { + psError(PS_ERR_UNKNOWN, true, "-det_type is required"); + return false; + } + + psString filelevel = psMetadataLookupStr(&status, config->args, "-filelevel"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filelevel"); + return false; + } + if (!filelevel) { + psError(PS_ERR_UNKNOWN, true, "-filelevel is required"); + return false; + } + + psString workdir = psMetadataLookupStr(&status, config->args, "-workdir"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -workdir"); + return false; + } + + psString camera = psMetadataLookupStr(&status, config->args, "-inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst"); + return false; + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope"); + return false; + } + + psString exp_type = psMetadataLookupStr(&status, config->args, "-exp_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_type"); + return false; + } + + psString filter = psMetadataLookupStr(&status, config->args, "-filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter"); + return false; + } + + psF32 airmass_min = psMetadataLookupF32(&status, config->args, "-airmass_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass_min"); + return false; + } + + psF32 airmass_max = psMetadataLookupF32(&status, config->args, "-airmass_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass_max"); + return false; + } + + psF32 exp_time_min = psMetadataLookupF32(&status, config->args, "-exp_time_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time_min"); + return false; + } + + psF32 exp_time_max = psMetadataLookupF32(&status, config->args, "-exp_time_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time_max"); + return false; + } + + psF32 ccd_temp_min = psMetadataLookupF32(&status, config->args, "-ccd_temp_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp_min"); + return false; + } + + psF32 ccd_temp_max = psMetadataLookupF32(&status, config->args, "-ccd_temp_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp_max"); + return false; + } + + psF64 posang_min = psMetadataLookupF32(&status, config->args, "-posang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang_min"); + return false; + } + + psF64 posang_max = psMetadataLookupF32(&status, config->args, "-posang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang_max"); + return false; + } + + psF64 solang_min = psMetadataLookupF32(&status, config->args, "-solang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -solang_min"); + return false; + } + + psF64 solang_max = psMetadataLookupF32(&status, config->args, "-solang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -solang_max"); + return false; + } + + psTime *registered = NULL; + { + psString registeredStr = psMetadataLookupStr(&status, config->args, "-registered"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -registered"); + return false; + } + // pass through NULL as this is an optional field + if (registeredStr) { + registered = psTimeFromISO(registeredStr, PS_TIME_UTC); + } else { + registered = NULL; + } + } + + psTime *time_begin = NULL; + { + psString time_beginStr = psMetadataLookupStr(&status, config->args, "-time_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time_begin"); + return false; + } + // pass through NULL as this is an optional field + if (time_beginStr) { + time_begin = psTimeFromISO(time_beginStr, PS_TIME_UTC); + } else { + time_begin = NULL; + } + } + + psTime *time_end = NULL; + { + psString time_endStr = psMetadataLookupStr(&status, config->args, "-time_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time_end"); + return false; + } + // pass through NULL as this is an optional field + if (time_endStr) { + time_end = psTimeFromISO(time_endStr, PS_TIME_UTC); + } else { + time_end = NULL; + } + } + + psTime *use_begin = NULL; + { + psString use_beginStr = psMetadataLookupStr(&status, config->args, "-use_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -use_begin"); + return false; + } + // pass through NULL as this is an optional field + if (use_beginStr) { + use_begin = psTimeFromISO(use_beginStr, PS_TIME_UTC); + } else { + use_begin = NULL; + } + } + + psTime *use_end = NULL; + { + psString use_endStr = psMetadataLookupStr(&status, config->args, "-use_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -use_end"); + return false; + } + // pass through NULL as this is an optional field + if (use_endStr) { + use_end = psTimeFromISO(use_endStr, PS_TIME_UTC); + } else { + use_end = NULL; + } + } + + psString parent = psMetadataLookupStr(&status, config->args, "-parent"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -parent"); + return false; + } + + psString label = psMetadataLookupStr(&status, config->args, "-label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!detRunInsert(config->dbh, + 0, // det_id + 0, // the iteration is fixed at 0 + det_type, + "register", // mode + "register", // state + filelevel, + workdir, + camera, + telescope, + exp_type, + NULL, + filter, + airmass_min, + airmass_max, + exp_time_min, + exp_time_max, + ccd_temp_min, + ccd_temp_max, + posang_min, + posang_max, + registered, + time_begin, + time_end, + use_begin, + use_end, + solang_min, + solang_max, + label, // label + parent ? (psS64)atoll(parent) : 0 + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(registered); + return false; + } + psFree(registered); + + // print the new detRun + psS64 det_id = psDBLastInsertID(config->dbh); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id); + detRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check results + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 "(this should not happen)", det_id); + return false; + } + + if (!convertIdToStr(detRuns)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(detRuns); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, detRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detRuns); + return false; + } + psFree(detRuns); + + return true; +} + +static bool register_detrend_imfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + return false; + } + if (!class_id) { + psError(PS_ERR_UNKNOWN, true, "-class_id is required"); + return false; + } + + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + + // everything else is optional + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + if (!detRegisteredImfileInsert(config->dbh, + (psS64)atoll(det_id), + 0, // the iteration is fixed at 0 + class_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + 0 // fault code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static psS32 incrementIteration(pxConfig *config, psS64 det_id) +{ + // this function returns zero on error + PS_ASSERT_PTR_NON_NULL(config, 0); + + char *query = "UPDATE detRun SET iteration = iteration + 1 WHERE det_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to increment iteration for det_id %" PRId64, det_id); + return 0; + } + + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return 0; + } + + psArray *detRuns = detRunSelectRowObjects(config->dbh, where, 0); + psFree(where); + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "no detRun rows found"); + return 0; + } + + // sanity check the database + if (psArrayLength(detRuns) != 1) { + // this should no happen + psAbort( "database query return too many rows (this should not happen"); + } + + psS32 newIteration = ((detRunRow *)detRuns->data[0])->iteration; + psFree(detRuns); + + return newIteration; +} + +static bool setDetRunState(pxConfig *config, psS64 det_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "drop", 5) == 0) + || (strncmp(state, "register", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid detRun state: %s", state); + return false; + } + + char *query = "UPDATE detRun SET state = '%s' WHERE det_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64, det_id); + return false; + } + + return true; +} + +static bool isValidMode(pxConfig *config, const char *mode) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(mode, false); + + // check that state is a valid string value + if (!( + (strncmp(mode, "master", 7) == 0) + || (strncmp(mode, "verify", 7) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid detRun mode: %s", mode); + return false; + } + + return true; +} Index: trunk/ippTools/src/caltoolConfig.c =================================================================== --- trunk/ippTools/src/caltoolConfig.c (revision 15537) +++ trunk/ippTools/src/caltoolConfig.c (revision 15537) @@ -0,0 +1,1237 @@ +/* + * dettoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "dettool.h" + +pxConfig *dettoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *time = psTimeGetNow(PS_TIME_TAI); + psString now = psTimeToISO(time); + psFree(time); + + // -pending + psMetadata *pendingArgs = psMetadataAlloc(); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_id", 0, + "search by exposure ID", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type", 0, + "search by exposure type", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst", 0, + "search by camera", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope", 0, + "search by telescope", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter", 0, + "search by filter", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-uri", 0, + "search by URL", NULL); + psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -definebytag + psMetadata *definebytagArgs = psMetadataAlloc(); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-exp_id", + PS_META_DUPLICATE_OK, + "include this exposure (multiple OK, required)", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-det_type", 0, + "define the type of detrend run (required)", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-mode", 0, + "define the mode of this detrend run", "master"); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-filelevel", 0, + "define filelevel", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-workdir", 0, + "define workdir (required)", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-inst", 0, + "define camera", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-telescope", 0, + "define telescope", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-exp_type", 0, + "define exposure type", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-filter", 0, + "define filter ", NULL); + psMetadataAddF32(definebytagArgs, PS_LIST_TAIL, "-airmass_min", 0, + "define min airmass", NAN); + psMetadataAddF32(definebytagArgs, PS_LIST_TAIL, "-airmass_max", 0, + "define max airmass", NAN); + psMetadataAddF32(definebytagArgs, PS_LIST_TAIL, "-exp_time_min", 0, + "define min exposure time", NAN); + psMetadataAddF32(definebytagArgs, PS_LIST_TAIL, "-exp_time_max", 0, + "define max exposure time", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, + "define min ccd tempature", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, + "define max ccd tempature", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-posang_min", 0, + "define min rotator position angle", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-posang_max", 0, + "define max rotator position angle", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-solang_min", 0, + "define min solar angle", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-solang_max", 0, + "define max solar angle", NAN); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-registered", 0, + "time detrend run was registered", now); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-time_begin", 0, + "detrend applyes to exposures taken during this peroid", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-time_end", 0, + "detrend applyes to exposures taken during this peroid", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-use_begin", 0, + "start of detrend run applicable peroid", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-use_end", 0, + "end of detrend run applicable peroid", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-reduction", 0, + "define reduction class for processing", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-label", 0, + "define detrun label", NULL); + psMetadataAddBool(definebytagArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-det_type", 0, + "define the type of detrend run (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-mode", 0, + "define the mode of this detrend run", "master"); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filelevel", 0, + "define filelevel", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, + "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-inst", 0, + "define camera (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-telescope", 0, + "define telescope", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filter", 0, + "define filter ", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-airmass_min", 0, + "define min airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-airmass_max", 0, + "define max airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_min", 0, + "define min exposure time", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_max", 0, + "define max exposure time", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, + "define min ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, + "define max ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_min", 0, + "define min rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_max", 0, + "define max rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-solang_min", 0, + "define min solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-solang_max", 0, + "define max solar angle", NAN); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, + "time detrend run was registered", now); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-time_begin", 0, + "detrend applyes to exposures taken during this peroid", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-time_end", 0, + "detrend applyes to exposures taken during this peroid", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-use_begin", 0, + "start of detrend run applicable peroid", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-use_end", 0, + "end of detrend run applicable peroid", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_type", 0, + "search for exp_type", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_inst", 0, + "search for camera", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_telescope", 0, + "search for telescope", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", 0, + "search for filter", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_uri", 0, + "search for uri", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_dateobs_begin", 0, + "search for exposures by time (>=)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_dateobs_end", 0, + "search for exposures by time (<)", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_airmass_min", 0, + "define min airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_airmass_max", 0, + "define max airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_sat_pixel_frac_max", 0, + "define max fraction of saturated pixels", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_min", 0, + "define min exposure time", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_max", 0, + "define max exposure time", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_ccd_temp_min", 0, + "define min ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_ccd_temp_max", 0, + "define max ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_min", 0, + "define min rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_max", 0, + "define max rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_min", 0, + "define min solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_max", 0, + "define max solar angle", NAN); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, + "print the exposures that would be included in the detrend run and exit", false); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, + "define reduction class for processing", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, + "define detrun label", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -definebydetrun + psMetadata *definebydetrunArgs = psMetadataAlloc(); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-det_id", 0, + "det ID to base a new detRun on (required)", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_det_type", 0, + "define the type of detrend run", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_mode", 0, + "define the mode of this detrend run", "master"); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_exp_type", 0, + "define exposure type", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_filelevel", 0, + "define filelevel", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_workdir", 0, + "define workdir", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_filter", 0, + "define filter ", NULL); + psMetadataAddF32(definebydetrunArgs, PS_LIST_TAIL, "-set_airmass_min", 0, + "define airmass", NAN); + psMetadataAddF32(definebydetrunArgs, PS_LIST_TAIL, "-set_airmass_max", 0, + "define airmass", NAN); + psMetadataAddF32(definebydetrunArgs, PS_LIST_TAIL, "-set_exp_time_min", 0, + "define exposure time", NAN); + psMetadataAddF32(definebydetrunArgs, PS_LIST_TAIL, "-set_exp_time_max", 0, + "define exposure time", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_ccd_temp_min", 0, + "define ccd tempature", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_ccd_temp_max", 0, + "define ccd tempature", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_posang_min", 0, + "define rotator position angle", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_posang_max", 0, + "define rotator position angle", NAN); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_registered", 0, + "time detrend run was registered", now); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_time_begin", 0, + "start of peroid to apply detrend too", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_time_end", 0, + "end of peroid to apply detrend too", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_use_begin", 0, + "start of detrend run applicable peroid", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_use_end", 0, + "end of detrend run applicable peroid", NULL); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_solang_min", 0, + "define solar angle", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_solang_max", 0, + "define solar angle", NAN); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-filter_input_begin", 0, + "filter input detrun exp to be in this peroid", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-filter_input_end", 0, + "filter input detrun exp to be in this peroid", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_reduction", 0, + "define reduction class for processing", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_label", 0, + "define detrun label", NULL); + psMetadataAddBool(definebydetrunArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -runs + psMetadata *runsArgs = psMetadataAlloc(); + psMetadataAddStr(runsArgs, PS_LIST_TAIL, "-det_type", 0, + "search for type of detrend run", NULL); + psMetadataAddBool(runsArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + psMetadataAddStr(runsArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID", NULL); + + // -childlessrun + psMetadata *childlessrunArgs = psMetadataAlloc(); + psMetadataAddStr(childlessrunArgs, PS_LIST_TAIL, "-det_type", 0, + "search for type of detrend run", NULL); + psMetadataAddU64(childlessrunArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(childlessrunArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -input + psMetadata *inputArgs = psMetadataAlloc(); + psMetadataAddStr(inputArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID", NULL); + psMetadataAddS32(inputArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddStr(inputArgs, PS_LIST_TAIL, "-exp_id", 0, + "search for exp ID", NULL); + psMetadataAddBool(inputArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -raw + psMetadata *rawArgs = psMetadataAlloc(); + psMetadataAddBool(rawArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -toprocessedimfile + psMetadata *toprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(toprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID", NULL); + psMetadataAddStr(toprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, + "search for exp ID", NULL); + psMetadataAddStr(toprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search for class ID", NULL); + psMetadataAddU64(toprocessedimfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(toprocessedimfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addprocessedimfile + psMetadata *addprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, + "define exp ID (required)", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "define class ID (required)", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-uri", 0, + "define URI (required)", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-recip", 0, + "define recipe (required)", NULL); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background (required)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev (required)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_0", 0, + "define fringe slope (0th term)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_1", 0, + "define fringe slope (1st term)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_2", 0, + "define fringe slope (2nd term)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_1", 0, + "define user statistic (1)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_2", 0, + "define user statistic (2)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_3", 0, + "define user statistic (3)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_4", 0, + "define user statistic (4)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_5", 0, + "define user statistic (5)", NAN); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + + // -processedimfile + psMetadata *processedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, + "search for exp ID", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search for class ID", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-select_state", 0, + "search for state", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-select_mode", 0, + "search for mode", NULL); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-chip", 0, + "restrict results to completed 'chip' sets", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-exp", 0, + "restrict results to complete 'exposures'", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-included", 0, + "restrict results to exposures 'includeded' in the current iteration", false); + psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -revertprocessedimfile + psMetadata *revertprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID (required)", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, + "search by exposure ID", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search by class ID", NULL); + psMetadataAddS16(revertprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, + "search by fault code", 0); + + // -toprocessedexp + psMetadata *toprocessedexpArgs = psMetadataAlloc(); + psMetadataAddU64(toprocessedexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(toprocessedexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addproccessedexp + psMetadata *addprocessedexpArgs = psMetadataAlloc(); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-recip", 0, + "define recipe (required)", NULL); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background (required)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev (required)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_0", 0, + "define fringe slope (0th term)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_1", 0, + "define fringe slope (1st term)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_2", 0, + "define fringe slope (2nd term)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_1", 0, + "define user statistic (1)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_2", 0, + "define user statistic (2)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_3", 0, + "define user statistic (3)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_4", 0, + "define user statistic (4)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_5", 0, + "define user statistic (5)", NAN); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + + // -proccessedexp + psMetadata *processedexpArgs = psMetadataAlloc(); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_id", 0, + "define detrend ID", NULL); + psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -revertprocessedexp + psMetadata *revertprocessedexpArgs = psMetadataAlloc(); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, + "search by detrend ID (required)", NULL); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, + "search by exposure ID", NULL); + psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-code", 0, + "search by fault code", 0); + + // -tostacked + psMetadata *tostackedArgs = psMetadataAlloc(); + psMetadataAddU64(tostackedArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(tostackedArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addstacked + psMetadata *addstackedArgs = psMetadataAlloc(); + psMetadataAddStr(addstackedArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddS32(addstackedArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddStr(addstackedArgs, PS_LIST_TAIL, "-class_id", 0, + "define class ID (required)", NULL); + psMetadataAddStr(addstackedArgs, PS_LIST_TAIL, "-recip", 0, + "define recipe (required)", NULL); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background (required)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev (required)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_1", 0, + "define user statistic (1)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_2", 0, + "define user statistic (2)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_3", 0, + "define user statistic (3)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_4", 0, + "define user statistic (4)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_5", 0, + "define user statistic (5)", NAN); + psMetadataAddStr(addstackedArgs, PS_LIST_TAIL, "-uri", 0, + "define URI (required)", NULL); + psMetadataAddS16(addstackedArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + + // -stacked + psMetadata *stackedArgs = psMetadataAlloc(); + psMetadataAddStr(stackedArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID", NULL); + psMetadataAddS32(stackedArgs, PS_LIST_TAIL, "-iteration", 0, + "search for iteration number", 0); + psMetadataAddStr(stackedArgs, PS_LIST_TAIL, "-class_id", 0, + "search for class ID", NULL); + psMetadataAddStr(stackedArgs, PS_LIST_TAIL, "-recip", 0, + "search for recipe", NULL); + psMetadataAddU64(stackedArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(stackedArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(stackedArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -revertstacked + psMetadata *revertstackedArgs= psMetadataAlloc(); + psMetadataAddStr(revertstackedArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID (required)", NULL); + psMetadataAddS32(revertstackedArgs, PS_LIST_TAIL, "-iteration", 0, + "search by iteration number", 0); + psMetadataAddStr(revertstackedArgs, PS_LIST_TAIL, "-class_id", 0, + "search by class ID", NULL); + psMetadataAddS16(revertstackedArgs, PS_LIST_TAIL, "-code", 0, + "search by fault code", 0); + + // -tonormalize + psMetadata *tonormalizeArgs = psMetadataAlloc(); + psMetadataAddU64(tonormalizeArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(tonormalizeArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addnormalizedstat + psMetadata *addnormstatArgs = psMetadataAlloc(); + psMetadataAddStr(addnormstatArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddS32(addnormstatArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddStr(addnormstatArgs, PS_LIST_TAIL, "-class_id", 0, + "define class ID (required)", NULL); + psMetadataAddF32(addnormstatArgs, PS_LIST_TAIL, "-norm", 0, + "define normal value (required)", NAN); + psMetadataAddS16(addnormstatArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + + // -normalizedstat + psMetadata *normalizedstatArgs = psMetadataAlloc(); + psMetadataAddStr(normalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, + "search by detrend ID", NULL); + psMetadataAddS32(normalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, + "search by iteration number", 0); + psMetadataAddStr(normalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, + "search by class ID", NULL); + psMetadataAddU64(normalizedstatArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(normalizedstatArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(normalizedstatArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -revertnormalizedstat + psMetadata *revertnormalizedstatArgs= psMetadataAlloc(); + psMetadataAddStr(revertnormalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, + "search by detrend ID (required)", NULL); + psMetadataAddS32(revertnormalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, + "search by iteration number", 0); + psMetadataAddStr(revertnormalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, + "search by class ID", NULL); + psMetadataAddS16(revertnormalizedstatArgs, PS_LIST_TAIL, "-code", 0, + "search by fault code", 0); + + // -tonormstat + psMetadata *tonormstatArgs = psMetadataAlloc(); + psMetadataAddU64(tonormstatArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(tonormstatArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addnormalizedimfile + psMetadata *addnormalizedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(addnormalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddS32(addnormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddStr(addnormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "define class ID (required)", NULL); + psMetadataAddStr(addnormalizedimfileArgs, PS_LIST_TAIL, "-uri", 0, + "define URI (required)", NULL); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_1", 0, + "define user statistic (1)", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_2", 0, + "define user statistic (2)", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_3", 0, + "define user statistic (3)", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_4", 0, + "define user statistic (4)", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_5", 0, + "define user statistic (5)", NAN); + psMetadataAddStr(addnormalizedimfileArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + psMetadataAddS16(addnormalizedimfileArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + + // -normalizedimfile + psMetadata *normalizedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(normalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID", NULL); + psMetadataAddS32(normalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, + "search for iteration number", 0); + psMetadataAddStr(normalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search for class ID", NULL); + psMetadataAddStr(normalizedimfileArgs, PS_LIST_TAIL, "-recip", 0, + "search for recipe", NULL); + psMetadataAddU64(normalizedimfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(normalizedimfileArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(normalizedimfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -revertnormalizedimfile + psMetadata *revertnormalizedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(revertnormalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, + "search by detrend ID (required)", NULL); + psMetadataAddS32(revertnormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, + "search by iteration number", 0); + psMetadataAddStr(revertnormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search by class ID", NULL); + psMetadataAddS16(revertnormalizedimfileArgs, PS_LIST_TAIL, "-code", 0, + "search by fault code", 0); + + // -tonormalizedexp + psMetadata *tonormalizedexpArgs = psMetadataAlloc(); + psMetadataAddU64(tonormalizedexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(tonormalizedexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addnormalizedexp + psMetadata *addnormalizedexpArgs = psMetadataAlloc(); + psMetadataAddStr(addnormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddS32(addnormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddStr(addnormalizedexpArgs, PS_LIST_TAIL, "-recip", 0, + "search for recipe (required)", NULL); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background (required)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev (required)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_1", 0, + "define user statistic (1)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_2", 0, + "define user statistic (2)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_3", 0, + "define user statistic (3)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_4", 0, + "define user statistic (4)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_5", 0, + "define user statistic (5)", NAN); + + psMetadataAddStr(addnormalizedexpArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + psMetadataAddS16(addnormalizedexpArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + + // -normalizedexp + psMetadata *normalizedexpArgs = psMetadataAlloc(); + psMetadataAddStr(normalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID", NULL); + psMetadataAddS32(normalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddStr(normalizedexpArgs, PS_LIST_TAIL, "-recip", 0, + "search for recipe", NULL); + psMetadataAddF64(normalizedexpArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background", NAN); + psMetadataAddF64(normalizedexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev", NAN); + psMetadataAddF64(normalizedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddStr(normalizedexpArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + psMetadataAddU64(normalizedexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(normalizedexpArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(normalizedexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -revertnormalizedexp + psMetadata *revertnormalizedexpArgs = psMetadataAlloc(); + psMetadataAddStr(revertnormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, + "search by detrend ID (required)", NULL); + psMetadataAddS32(revertnormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, + "search by iteration number", 0); + psMetadataAddS16(revertnormalizedexpArgs, PS_LIST_TAIL, "-code", 0, + "search by fault code", 0); + + // -toresidimfile + psMetadata *toresidimfileArgs = psMetadataAlloc(); + psMetadataAddU64(toresidimfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(toresidimfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -toresidexp + psMetadata *toresidexpArgs = psMetadataAlloc(); + psMetadataAddU64(toresidexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(toresidexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addresidimfile + psMetadata *addresidimfileArgs = psMetadataAlloc(); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddS32(addresidimfileArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-exp_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "define class ID (required)", NULL); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-uri", 0, + "define resid file URI (required)", NULL); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-recip", 0, + "define recipe (required)", NULL); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background (required)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev (required)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg_skewness", 0, + "define exposure background skewness", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg_kurtosis", 0, + "define exposure background kurtosis", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bin_stdev", 0, + "define exposure background binned stdev", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_0", 0, + "define fringe slope (0th term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_1", 0, + "define fringe slope (1st term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_2", 0, + "define fringe slope (2nd term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_resid_0", 0, + "define fringe residual (0th term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_resid_1", 0, + "define fringe residual (1st term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_resid_2", 0, + "define fringe residual (2nd term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_1", 0, + "define user statistic (1)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_2", 0, + "define user statistic (2)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_3", 0, + "define user statistic (3)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_4", 0, + "define user statistic (4)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_5", 0, + "define user statistic (5)", NAN); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + psMetadataAddS16(addresidimfileArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + + // -residimfile + psMetadata *residimfileArgs = psMetadataAlloc(); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID", NULL); + psMetadataAddS32(residimfileArgs, PS_LIST_TAIL, "-iteration", 0, + "search for iteration number", 0); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-exp_id", 0, + "define detrend ID", NULL); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search for class ID", NULL); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-recip", 0, + "search for recipe", NULL); + psMetadataAddU64(residimfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-select_state", 0, + "search for state", NULL); + + // -revertresidimfile + psMetadata *revertresidimfileArgs = psMetadataAlloc(); + psMetadataAddStr(revertresidimfileArgs, PS_LIST_TAIL, "-det_id", 0, + "search by detrend ID (required)", NULL); + psMetadataAddS32(revertresidimfileArgs, PS_LIST_TAIL, "-iteration", 0, + "search by iteration number", 0); + psMetadataAddStr(revertresidimfileArgs, PS_LIST_TAIL, "-exp_id", 0, + "search by detrend ID", NULL); + psMetadataAddStr(revertresidimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search for class ID", NULL); + psMetadataAddS16(revertresidimfileArgs, PS_LIST_TAIL, "-code", 0, + "search by fault code", 0); + + // -addresidexp + psMetadata *addresidexpArgs = psMetadataAlloc(); + psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddS32(addresidexpArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-exp_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-recip", 0, + "define recipe (required)", NULL); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background (required)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev (required)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg_skewness", 0, + "define exposure background skewness", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg_kurtosis", 0, + "define exposure background kurtosis", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bin_stdev", 0, + "define exposure background binned stdev", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_0", 0, + "define fringe slope (0th term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_1", 0, + "define fringe slope (1st term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_2", 0, + "define fringe slope (2nd term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_resid_0", 0, + "define fringe residual (0th term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_resid_1", 0, + "define fringe residual (1st term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_resid_2", 0, + "define fringe residual (2nd term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_1", 0, + "define user statistic (1)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_2", 0, + "define user statistic (2)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_3", 0, + "define user statistic (3)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_4", 0, + "define user statistic (4)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_5", 0, + "define user statistic (5)", NAN); + psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + psMetadataAddS16(addresidexpArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + psMetadataAddBool(addresidexpArgs, PS_LIST_TAIL, "-reject", 0, + "exposure is not to be stacked in the next iteration", false); + + // -residexp + psMetadata *residexpArgs = psMetadataAlloc(); + psMetadataAddStr(residexpArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID", NULL); + psMetadataAddS32(residexpArgs, PS_LIST_TAIL, "-iteration", 0, + "search for iteration number", 0); + psMetadataAddStr(residexpArgs, PS_LIST_TAIL, "-exp_id", 0, + "search for exp ID", NULL); + psMetadataAddStr(residexpArgs, PS_LIST_TAIL, "-recip", 0, + "search for recipe", NULL); + psMetadataAddU64(residexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(residexpArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(residexpArgs, PS_LIST_TAIL, "-reject", 0, + "search for acceptable residuals", false); + psMetadataAddBool(residexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -revertresidexp + psMetadata *revertresidexpArgs = psMetadataAlloc(); + psMetadataAddStr(revertresidexpArgs, PS_LIST_TAIL, "-det_id", 0, + "search by detrend ID (required)", NULL); + psMetadataAddS32(revertresidexpArgs, PS_LIST_TAIL, "-iteration", 0, + "search by iteration number", 0); + psMetadataAddStr(revertresidexpArgs, PS_LIST_TAIL, "-exp_id", 0, + "search by detrend ID", NULL); + psMetadataAddS16(revertresidexpArgs, PS_LIST_TAIL, "-code", 0, + "search by fault code", 0); + + // -todetrunsummary + psMetadata *todetrunsummaryArgs = psMetadataAlloc(); + psMetadataAddBool(todetrunsummaryArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + psMetadataAddU64(todetrunsummaryArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + + // -updateresidexp + psMetadata *updateresidexpArgs = psMetadataAlloc(); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID", NULL); + psMetadataAddS32(updateresidexpArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-exp_id", 0, + "define exp ID", NULL); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-recip", 0, + "define recipe", NULL); + psMetadataAddF64(updateresidexpArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background", NAN); + psMetadataAddF64(updateresidexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev", NAN); + psMetadataAddF64(updateresidexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + psMetadataAddBool(updateresidexpArgs, PS_LIST_TAIL, "-reject", 0, + "exposure is not to be stacked in the next iteration", false); + + // -adddetrunsummary + psMetadata *adddetrunsummaryArgs = psMetadataAlloc(); + psMetadataAddStr(adddetrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddS32(adddetrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddF64(adddetrunsummaryArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background (required)", NAN); + psMetadataAddF64(adddetrunsummaryArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev (required)", NAN); + psMetadataAddF64(adddetrunsummaryArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddS16(adddetrunsummaryArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + psMetadataAddBool(adddetrunsummaryArgs, PS_LIST_TAIL, "-accept", 0, + "declare that this detrun iteration is accepted as a master", false); + psMetadataAddBool(adddetrunsummaryArgs, PS_LIST_TAIL, "-again", 0, + "start a new iteration of this detrend run", false); + + // -detrunsummary + psMetadata *detrunsummaryArgs = psMetadataAlloc(); + psMetadataAddStr(detrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID", NULL); + psMetadataAddU64(detrunsummaryArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddS32(detrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, + "search for iteration number", 0); + psMetadataAddBool(detrunsummaryArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(detrunsummaryArgs, PS_LIST_TAIL, "-reject", 0, + "search for acceptable residuals", false); + psMetadataAddBool(detrunsummaryArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -revertdetrunsummary + psMetadata *revertdetrunsummaryArgs = psMetadataAlloc(); + psMetadataAddStr(revertdetrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, + "search by detrend ID (required)", NULL); + psMetadataAddS32(revertdetrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, + "search by iteration number", 0); + psMetadataAddS16(revertdetrunsummaryArgs, PS_LIST_TAIL, "-code", 0, + "search by fault code", 0); + + // -updatedetrun + psMetadata *updatedetrunArgs = psMetadataAlloc(); + psMetadataAddStr(updatedetrunArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend master for detrend ID (required)", NULL); + psMetadataAddBool(updatedetrunArgs, PS_LIST_TAIL, "-again", 0, + "start a new iteration of this detrend run", false); + psMetadataAddStr(updatedetrunArgs, PS_LIST_TAIL, "-state", 0, + "set the state of this detrend run", false); + + // -rerun + psMetadata *rerunArgs = psMetadataAlloc(); + psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend master for detrend ID (required)", NULL); + psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-exp_id", PS_META_DUPLICATE_OK, + "include this exposure (multiple OK, required)", NULL); + + // -register_detrend + psMetadata *register_detrendArgs = psMetadataAlloc(); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-det_type", 0, + "define the type of detrend run (required)", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-mode", 0, + "define the mode of this detrend run", "master"); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-filelevel", 0, + "define filelevel (required)", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-workdir", 0, + "define workdir", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-inst", 0, + "define camera", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-telescope", 0, + "define telescope", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-exp_type", 0, + "define exposure type", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-filter", 0, + "define filter ", NULL); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-airmass_min", 0, + "define min airmass", NAN); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-airmass_max", 0, + "define max airmass", NAN); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-exp_time_min", 0, + "define min exposure time", NAN); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-exp_time_max", 0, + "define max exposure time", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, + "define min ccd tempature", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, + "define max ccd tempature", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-posang_min", 0, + "define min rotator position angle", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-posang_max", 0, + "define max rotator position angle", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-solang_min", 0, + "define min solar angle", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-solang_max", 0, + "define max solar angle", NAN); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-registered", 0, + "time detrend run was registered", now); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-time_begin", 0, + "detrend applyes to exposures taken during this peroid", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-time_end", 0, + "detrend applyes to exposures taken during this peroid", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-use_begin", 0, + "start of detrend run applicable peroid", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-use_end", 0, + "end of detrend run applicable peroid", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-parent", 0, + "define parent det_id", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-label", 0, + "define detrun label", NULL); + psMetadataAddBool(register_detrendArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -register_detrend_imfile + psMetadata *register_detrend_imfileArgs = psMetadataAlloc(); + psMetadataAddStr(register_detrend_imfileArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddStr(register_detrend_imfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search for class ID (required)", NULL); + psMetadataAddStr(register_detrend_imfileArgs, PS_LIST_TAIL, "-uri", 0, + "define resid file URI (required)", NULL); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-user_1", 0, + "define user statistic (1)", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-user_2", 0, + "define user statistic (2)", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-user_3", 0, + "define user statistic (3)", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-user_4", 0, + "define user statistic (4)", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-user_5", 0, + "define user statistic (5)", NAN); + psMetadataAddStr(register_detrend_imfileArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXTOOL_ADD_MODE("-pending", "list active detruns", DETTOOL_MODE_PENDING, pendingArgs); + PXTOOL_ADD_MODE("-definebytag", "", DETTOOL_MODE_DEFINEBYTAG, definebytagArgs); + PXTOOL_ADD_MODE("-definebyquery", "", DETTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXTOOL_ADD_MODE("-definebydetrun", "", DETTOOL_MODE_DEFINEBYDETRUN, definebydetrunArgs); + PXTOOL_ADD_MODE("-raw", "", DETTOOL_MODE_RAW, rawArgs); + PXTOOL_ADD_MODE("-runs", "", DETTOOL_MODE_RUNS, runsArgs); + PXTOOL_ADD_MODE("-childlessrun", "", DETTOOL_MODE_CHILDLESSRUN, childlessrunArgs); + PXTOOL_ADD_MODE("-input", "", DETTOOL_MODE_INPUT, inputArgs); + PXTOOL_ADD_MODE("-toprocessedimfile", "", DETTOOL_MODE_TOPROCESSEDIMFILE, toprocessedimfileArgs); + PXTOOL_ADD_MODE("-addprocessedimfile", "", DETTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); + PXTOOL_ADD_MODE("-processedimfile", "", DETTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); + PXTOOL_ADD_MODE("-revertprocessedimfile", "", DETTOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileArgs); + PXTOOL_ADD_MODE("-toprocessedexp", "", DETTOOL_MODE_TOPROCESSEDEXP, toprocessedexpArgs); + PXTOOL_ADD_MODE("-addprocessedexp", "", DETTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs); + PXTOOL_ADD_MODE("-revertprocessedexp", "", DETTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpArgs); + PXTOOL_ADD_MODE("-processedexp", "", DETTOOL_MODE_PROCESSEDEXP, processedexpArgs); + PXTOOL_ADD_MODE("-tostacked", "", DETTOOL_MODE_TOSTACKED, tostackedArgs); + PXTOOL_ADD_MODE("-addstacked", "", DETTOOL_MODE_ADDSTACKED, addstackedArgs); + PXTOOL_ADD_MODE("-stacked", "", DETTOOL_MODE_STACKED, stackedArgs); + PXTOOL_ADD_MODE("-revertstacked", "", DETTOOL_MODE_REVERTSTACKED, revertstackedArgs); + PXTOOL_ADD_MODE("-tonormalize", "", DETTOOL_MODE_TONORMALIZE, tonormalizeArgs); + PXTOOL_ADD_MODE("-addnormalizedstat", "", DETTOOL_MODE_ADDNORMALIZEDSTAT, addnormstatArgs); + PXTOOL_ADD_MODE("-normalizedstat", "", DETTOOL_MODE_NORMALIZEDSTAT, normalizedstatArgs); + PXTOOL_ADD_MODE("-revertnormalizedstat", "", DETTOOL_MODE_REVERTNORMALIZEDSTAT, revertnormalizedstatArgs); + + PXTOOL_ADD_MODE("-tonormalizedstat", "", DETTOOL_MODE_TONORMALIZEDSTAT, tonormstatArgs); + PXTOOL_ADD_MODE("-addnormalizedimfile", "", DETTOOL_MODE_ADDNORMALIZEDIMFILE,addnormalizedimfileArgs); + PXTOOL_ADD_MODE("-normalizedimfile","", DETTOOL_MODE_NORMALIZEDIMFILE, normalizedimfileArgs); + PXTOOL_ADD_MODE("-revertnormalizedimfile","", DETTOOL_MODE_REVERTNORMALIZEDIMFILE, revertnormalizedimfileArgs); + PXTOOL_ADD_MODE("-tonormalizedexp", "", DETTOOL_MODE_TONORMALIZEDEXP, tonormalizedexpArgs); + PXTOOL_ADD_MODE("-addnormalizedexp", "", DETTOOL_MODE_ADDNORMALIZEDEXP, addnormalizedexpArgs); + PXTOOL_ADD_MODE("-normalizedexp", "", DETTOOL_MODE_NORMALIZEDEXP, normalizedexpArgs); + PXTOOL_ADD_MODE("-revertnormalizedexp","", DETTOOL_MODE_REVERTNORMALIZEDEXP, revertnormalizedexpArgs); + PXTOOL_ADD_MODE("-toresidimfile", "", DETTOOL_MODE_TORESIDIMFILE, toresidimfileArgs); + PXTOOL_ADD_MODE("-addresidimfile", "", DETTOOL_MODE_ADDRESIDIMFILE, addresidimfileArgs); + PXTOOL_ADD_MODE("-residimfile", "", DETTOOL_MODE_RESIDIMFILE, residimfileArgs); + PXTOOL_ADD_MODE("-revertresidimfile", "", DETTOOL_MODE_REVERTRESIDIMFILE, revertresidimfileArgs); + PXTOOL_ADD_MODE("-toresidexp", "", DETTOOL_MODE_TORESIDEXP, toresidexpArgs); + PXTOOL_ADD_MODE("-addresidexp", "", DETTOOL_MODE_ADDRESIDEXP, addresidexpArgs); + PXTOOL_ADD_MODE("-residexp", "", DETTOOL_MODE_RESIDEXP, residexpArgs); + PXTOOL_ADD_MODE("-revertresidexp", "", DETTOOL_MODE_REVERTRESIDEXP, revertresidexpArgs); + PXTOOL_ADD_MODE("-todetrunsummary", "", DETTOOL_MODE_TODETRUNSUMMARY, todetrunsummaryArgs); + PXTOOL_ADD_MODE("-updateresidexp", "", DETTOOL_MODE_UPDATERESIDEXP,updateresidexpArgs); + PXTOOL_ADD_MODE("-adddetrunsummary", "", DETTOOL_MODE_ADDDETRUNSUMMARY,adddetrunsummaryArgs); + PXTOOL_ADD_MODE("-detrunsummary", "", DETTOOL_MODE_DETRUNSUMMARY,detrunsummaryArgs); + PXTOOL_ADD_MODE("-revertdetrunsummary", "", DETTOOL_MODE_REVERTDETRUNSUMMARY, revertdetrunsummaryArgs); + PXTOOL_ADD_MODE("-updatedetrun", "", DETTOOL_MODE_UPDATEDETRUN, updatedetrunArgs); + PXTOOL_ADD_MODE("-rerun", "", DETTOOL_MODE_RERUN, rerunArgs); + PXTOOL_ADD_MODE("-register_detrend", "", DETTOOL_MODE_REGISTER_DETREND, register_detrendArgs); + PXTOOL_ADD_MODE("-register_detrend_imfile", "", DETTOOL_MODE_REGISTER_DETREND_IMFILE, register_detrend_imfileArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // setup search criterion +#define addWhereStr(name) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, #name, 0, "==", str)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + + // generate SQL where clause + config->where = psMetadataAlloc(); + + addWhereStr(det_id); + { + // XXX this is broke because you can't specify -iteration 0 + psMetadataItem *item = psMetadataLookup(config->args, "-iteration"); + if (item && psStrcasestr(item->comment, "found")) { + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, "iteration", 0, "==", item->data.S32)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(config); + return NULL; + } + } + } + addWhereStr(det_type); + addWhereStr(exp_id); + addWhereStr(class_id); + // convert '-inst' to 'camera' + { + psString str = NULL; + bool status = false; + if ((str = psMetadataLookupStr(&status, config->args, "-inst"))) { + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, "camera", 0, "==", str)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); + psFree(config); + return NULL; + } + } + } + addWhereStr(telescope); + addWhereStr(exp_type); + { + int n = 0; + bool status = false; + if ((n = psMetadataLookupS32(&status, config->args, "-imfiles"))) { + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, "imfiles", 0, "==", n)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item imfiles"); + psFree(config); + return NULL; + } + } + } + addWhereStr(filter); + addWhereStr(recipe); + { + int n = 0; + bool status = false; + if ((n = psMetadataLookupS32(&status, config->args, "-guide_version"))) { + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, "guide_version", 0, "==", n)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item guide_version"); + psFree(config); + return NULL; + } + } + } + { + bool boolean = false; + bool status = false; + + // map reject -> !accept + if ((boolean = psMetadataLookupBool(&status, config->args, "-reject"))) { + if (!psMetadataAddBool(config->where, PS_LIST_TAIL, "accept", 0, "==", !boolean)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item reject"); + psFree(config); + return NULL; + } + } + } + + // convert '-code' to 'fault' + { + psS16 fault = 0; + bool status = false; + if ((fault = psMetadataLookupS16(&status, config->args, "-code"))) { + if (!psMetadataAddS16(config->where, PS_LIST_TAIL, "fault", 0, "==", fault)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item fault"); + psFree(config); + return NULL; + } + } + } + + if (config->where->list->n < 1) { + psFree(config->where); + config->where = NULL; + } + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = pmConfigDB(config->modules); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +}