Index: branches/eam_branches/ipp-20100621/ippTools/src/Makefile.am =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/Makefile.am (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/Makefile.am (revision 28794) @@ -1,4 +1,5 @@ bin_PROGRAMS = \ addtool \ + bgtool \ caltool \ camtool \ @@ -48,4 +49,5 @@ noinst_HEADERS = \ addtool.h \ + bgtool.h \ caltool.h \ camtool.h \ @@ -117,4 +119,10 @@ addtoolConfig.c +bgtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +bgtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +bgtool_SOURCES = \ + bgtool.c \ + bgtoolConfig.c + caltool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) caltool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la Index: branches/eam_branches/ipp-20100621/ippTools/src/bgtool.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/bgtool.c (revision 28794) +++ branches/eam_branches/ipp-20100621/ippTools/src/bgtool.c (revision 28794) @@ -0,0 +1,1809 @@ +/* + * bgtool.c + * + * Copyright (C) 2006-2010 Joshua Hoblitt, Paul Price + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "bgtool.h" + +static bool definechipMode(pxConfig *config); +static bool updatechipMode(pxConfig *config); +static bool tochipMode(pxConfig *config); +static bool chipinputsMode(pxConfig *config); +static bool addchipMode(pxConfig *config); +static bool chipMode(pxConfig *config); +static bool advancechipMode(pxConfig *config); +static bool revertchipMode(pxConfig *config); +static bool definewarpMode(pxConfig *config); +static bool updatewarpMode(pxConfig *config); +static bool towarpMode(pxConfig *config); +static bool warpinputsMode(pxConfig *config); +static bool addwarpMode(pxConfig *config); +static bool warpMode(pxConfig *config); +static bool advancewarpMode(pxConfig *config); +static bool revertwarpMode(pxConfig *config); +static bool tocleanchipMode(pxConfig *config); +static bool cleanedchipMode(pxConfig *config); +static bool tocleanwarpMode(pxConfig *config); +static bool cleanedwarpMode(pxConfig *config); +static bool exportchipMode(pxConfig *config); +static bool importchipMode(pxConfig *config); +static bool exportwarpMode(pxConfig *config); +static bool importwarpMode(pxConfig *config); + +// Tables to import/export +typedef struct { + const char *name; // Table name + void* (*parse)(); // Parsing function + bool (*insert)(); // Insertion function +} tableData; +static const tableData chipTables[] = { + { "chipBackgroundRun", (void*)&chipBackgroundRunObjectFromMetadata, &chipBackgroundRunInsertObject }, + { "chipBackgroundImfile", (void*)&chipBackgroundImfileObjectFromMetadata, &chipBackgroundImfileInsertObject }, + { NULL, NULL, NULL } +}; +static const tableData warpTables[] = { + { "warpBackgroundRun", (void*)&warpBackgroundRunObjectFromMetadata, &warpBackgroundRunInsertObject }, + { "warpBackgroundSkyfile", (void*)&warpBackgroundSkyfileObjectFromMetadata, &warpBackgroundSkyfileInsertObject }, + { NULL, NULL, NULL } +}; + + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = bgtoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(BGTOOL_MODE_DEFINECHIP, definechipMode); + MODECASE(BGTOOL_MODE_UPDATECHIP, updatechipMode); + MODECASE(BGTOOL_MODE_TOCHIP, tochipMode); + MODECASE(BGTOOL_MODE_CHIPINPUTS, chipinputsMode); + MODECASE(BGTOOL_MODE_ADDCHIP, addchipMode); + MODECASE(BGTOOL_MODE_CHIP, chipMode); + MODECASE(BGTOOL_MODE_ADVANCECHIP, advancechipMode); + MODECASE(BGTOOL_MODE_REVERTCHIP, revertchipMode); + MODECASE(BGTOOL_MODE_DEFINEWARP, definewarpMode); + MODECASE(BGTOOL_MODE_UPDATEWARP, updatewarpMode); + MODECASE(BGTOOL_MODE_TOWARP, towarpMode); + MODECASE(BGTOOL_MODE_WARPINPUTS, warpinputsMode); + MODECASE(BGTOOL_MODE_ADDWARP, addwarpMode); + MODECASE(BGTOOL_MODE_WARP, warpMode); + MODECASE(BGTOOL_MODE_ADVANCEWARP, advancewarpMode); + MODECASE(BGTOOL_MODE_REVERTWARP, revertwarpMode); + MODECASE(BGTOOL_MODE_TOCLEANCHIP, tocleanchipMode); + MODECASE(BGTOOL_MODE_CLEANEDCHIP, cleanedchipMode); + MODECASE(BGTOOL_MODE_TOCLEANWARP, tocleanwarpMode); + MODECASE(BGTOOL_MODE_CLEANEDWARP, cleanedwarpMode); + MODECASE(BGTOOL_MODE_EXPORTCHIP, exportchipMode); + MODECASE(BGTOOL_MODE_IMPORTCHIP, importchipMode); + MODECASE(BGTOOL_MODE_EXPORTWARP, exportwarpMode); + MODECASE(BGTOOL_MODE_IMPORTWARP, importwarpMode); + + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// General functions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +static bool exportTables(pxConfig *config, // Configuration (with DB handle) + const char *filename, // Filename to which to write + const tableData tables[], // Tables to export (NULL terminated) + const psMetadata *where, // WHERE restrictions + bool clean // Write as cleaned? + ) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + FILE *file = fopen(filename, "w"); + if (!file) { + psError(PXTOOLS_ERR_SYS, true, "failed to open output file %s", filename); + return false; + } + + if (!pxExportVersion(config, file)) { + psError(psErrorCodeLast(), false, "failed to write dbversion to output file %s", filename); + return false; + } + + for (int i = 0; tables[i].name; i++) { + const char *name = tables[i].name; // Name of table + psString query = NULL; + psStringAppend(&query, "SELECT * FROM %s", name); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PXTOOLS_ERR_CONFIG, true, "no rows found"); + psFree(output); + return false; + } + + if (clean && + (strcmp(name, "chipBackgroundRun") == 0 || + strcmp(name, "warpBackgroundRun") == 0) && + !pxSetStateCleaned(name, "state", output)) { + psFree(output); + psError(psErrorCodeLast(), false, "pxSetStateClean failed for table %s", name); + return false; + } + + if (!ippdbPrintMetadatas(file, output, name, true)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + } + fclose(file); + + return true; +} + +static bool importTables(pxConfig *config, // Configuration + const char *filename, // + const tableData tables[] // Tables to read in + ) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + unsigned int badLines = 0; // Number of bad lines + psMetadata *input = psMetadataConfigRead(NULL, &badLines, filename, false); // Input file contents + if (!input) { + psError(psErrorCodeLast(), false, "Unable to parse input file %s", filename); + return false; + } + if (badLines > 0) { + psWarning("%d bad lines encountered when parsing %s", badLines, filename); + } + + if (!pxCheckImportVersion(config, input)) { + psError(psErrorCodeLast(), false, "pxCheckImportVersion failed"); + return false; + } + + // Import primary table + for (int i = 0; tables[i].name; i++) { + const char *name = tables[i].name; // Name of table + psMetadataItem *item = psMetadataLookup(input, name); // Item from input + psAssert(item, "%s not in input", name); + psAssert(item->type == PS_DATA_METADATA_MULTI, "%s not MULTI type", name); + psAssert(psListLength(item->data.list) == 1, "%s has multiple entries", name); + psMetadataItem *entry = psListGet(item->data.list, PS_LIST_HEAD); // Entry of interest + void *data = tables[i].parse(entry); // Parsed entry + if (!data) { + psError(PXTOOLS_ERR_CONFIG, false, "Unable to parse entry %s", name); + psFree(input); + return false; + } + if (!tables[0].insert(config->dbh, data)) { + psError(psErrorCodeLast(), false, "Unable to insert entry %s", name); + psFree(input); + return false; + } + } + + psFree(input); + + return true; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Functions for chip stage +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +static bool definechipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "LIKE"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); + pxAddLabelSearchArgs(config, where, "-label", "chipRun.label", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + // Get chip runs to promote to chipBackgroundRun + + psString query = pxDataGet("bgtool_definechip.sql"); // Query to execute + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nAND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (destreaked) { + psStringAppend(&query, " AND chipRun.magicked > 0"); + } + + if (!rerun) { + psStringAppend(&query, "\nAND chipBackgroundRun.chip_bg_id IS NULL"); + if (label) { + psStringAppend(&query, + "\nAND (chipBackgroundRun.label = '%s'" + " OR chipBackgroundRun.label IS NULL)", + label); + } + if (data_group) { + psStringAppend(&query, + "\nAND (chipBackgroundRun.data_group = '%s'" + " OR chipBackgroundRun.data_group IS NULL)", + data_group); + } + if (dist_group) { + psStringAppend(&query, + "\nAND (chipBackgroundRun.dist_group = '%s'" + " OR chipBackgroundRun.dist_group IS NULL)", + dist_group); + } + } + + if (!psDBTransaction(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); // Matching rows + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipRun", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + chipRunRow *row = chipRunObjectFromMetadata(md); + if (!row) { + psError(psErrorCodeLast(), false, "failed to convert metadata into fakeRun"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + if (!chipBackgroundRunInsert(config->dbh, 0, row->chip_id, "new", + workdir ? workdir : row->workdir, + label ? label : row->label, + data_group ? data_group : row->data_group, + dist_group ? dist_group : row->dist_group, + reduction ? reduction : row->reduction, + note ? note : row->note, + NULL, 0)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(row); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + return true; +} + +static bool updatechipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "chipBackgroundRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "chipBackgroundRun.data_group","LIKE"); + PXOPT_COPY_STR(config->args, where, "-dist_group", "chipBackgroundRun.dist_group","LIKE"); + pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "LIKE"); + PXOPT_COPY_TIME(config->args, where, "-registered_begin", "chipBackgroundRun.registered", ">="); + PXOPT_COPY_TIME(config->args, where, "-registered_end", "chipBackgroundRun.registered", "<"); + + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + if (destreaked) { + psMetadataAddS64(where, PS_LIST_TAIL, "chipBackgroundRun.magicked", PS_META_DUPLICATE_OK, ">", 0); + } + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE chipBackgroundRun"); + bool result = pxUpdateRun(config, where, &query, "chipBackgroundRun", "chip_bg_id", + "chipBackgroundImfile", true); + + psFree(query); + psFree(where); + + return result; +} + +static bool tochipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("bgtool_tochip.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + psString whereStr = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereStr, "\n AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQueryF(config->dbh, query, whereStr)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(whereStr); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "Unable to fetch result of query %s", query); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundRun", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool chipinputsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("bgtool_chipinputs.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundImfile", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + + // optional + + PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_ppbackground, config->args, "-ver_ppbackground", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); + + psString ver_code = pxMergeCodeVersions(ver_pslib, ver_psmodules); + ver_code = pxMergeCodeVersions(ver_code, ver_ppbackground); + ver_code = pxMergeCodeVersions(ver_code, ver_ppstats); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!chipBackgroundImfileInsert(config->dbh, chip_bg_id, class_id, path_base, magicked, dtime_script, + hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix, + maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + return true; +} + +static bool chipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "chipBackgroundImfile.class_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "LIKE"); + pxAddLabelSearchArgs(config, where, "-data_group", "chipBackgroundRun.data_group", "LIKE"); + pxAddLabelSearchArgs(config, where, "-dist_group", "chipBackgroundRun.data_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("bgtool_chip.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + psString magicWhere = NULL; + if (!pxmagicAddWhere(config, &magicWhere, "chipBackgroundImfile")) { + psError(psErrorCodeLast(), false, "pxMagicAddWhere failed"); + return false; + } + if (!pxspaceAddWhere(config, &magicWhere, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else if (!all && !magicWhere) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + if (magicWhere) { + psStringAppend(&query, "%s %s", psListLength(where->list) ? "AND" : "WHERE", magicWhere); + } + psFree(magicWhere); + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundImfile", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + } + psFree(output); + + return true; +} + + +static bool advancechipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chip_bg_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "label", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psString select = pxDataGet("bgtool_advancechip.sql"); + if (!select) { + psError(psErrorCodeLast(), false, "failed to retrieve SQL statement"); + return false; + } + + psString selectWhere = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&selectWhere, "\n AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&select, " %s", limitString); + psFree(limitString); + } + + if (!psDBTransaction(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, select, selectWhere)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(select); + psFree(selectWhere); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(select); + psFree(selectWhere); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + bool status = true; // Status of MD lookup + psS64 chip_bg_id = psMetadataLookupS64(&status, row, "chip_bg_id"); + if (!status) { + psError(PXTOOLS_ERR_PROG, true, "failed to look up value for chip_bg_id"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psS64 magicked = psMetadataLookupS64(&status, row, "magicked"); + if (!status) { + psError(PXTOOLS_ERR_PROG, true, "failed to look up value for magicked"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + if (!p_psDBRunQueryF(config->dbh, + "UPDATE chipBackgroundRun " + "SET state = 'full', magicked = %" PRId64 " " + " WHERE chip_bg_id = %" PRId64, + magicked, chip_bg_id)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + psS64 numUpdated = psDBAffectedRows(config->dbh); + if (numUpdated != 1) { + psError(PXTOOLS_ERR_PROG, true, "should have affected 1 row"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + psFree(output); + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + return true; +} + +static bool revertchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "chipBackgroundImfile.class_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "chipBackgroundImfile.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required"); + return false; + } + + psString query = pxDataGet("bgtool_revertchip.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + int numDeleted = psDBAffectedRows(config->dbh); + psLogMsg("bgtool", PS_LOG_INFO, "Deleted %d chipBackgroundImfiles", numDeleted); + + return true; +} + +static bool tocleanchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "=="); + + psString query = pxDataGet("bgtool_tocleanchip.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundRun", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + + return true; +} + +static bool cleanedchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); + + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + if (!pxIsValidCleanedState(state)) { + psError(PXTOOLS_ERR_CONFIG, true, "Invalid state: %s", state); + return false; + } + + if (!psListLength(where->list)) { + psError(PXTOOLS_ERR_CONFIG, true, "No search restrictions set."); + return false; + } + + psString query = pxDataGet("bgtool_cleanedchip.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, state)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + psFree(query); + + return true; +} + +static bool exportchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chip_bg_id", "=="); + + bool status = exportTables(config, outfile, chipTables, where, clean); + + psFree(where); + return status; +} + +static bool importchipMode(pxConfig *config) +{ + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); + + return importTables(config, infile, chipTables); +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Functions for warp stage +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +static bool definewarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "LIKE"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); + pxAddLabelSearchArgs(config, where, "-warp_label", "warpRun.label", "=="); + pxAddLabelSearchArgs(config, where, "-chip_label", "chipBackgroundRun.label", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + // Get warp runs to promote to warpBackgroundRun + + psString query = pxDataGet("bgtool_definewarp.sql"); // Query to execute + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (destreaked) { + psStringAppend(&query, " AND warpRun.magicked > 0"); + } + + if (!rerun) { + psStringAppend(&query, "\nAND warpBackgroundRun.warp_bg_id IS NULL"); + if (label) { + psStringAppend(&query, + "\nAND (warpBackgroundRun.label = '%s'" + " OR warpBackgroundRun.label IS NULL)", + label); + } + if (data_group) { + psStringAppend(&query, + "\nAND (warpBackgroundRun.data_group = '%s'" + " OR warpBackgroundRun.data_group IS NULL)", + data_group); + } + if (dist_group) { + psStringAppend(&query, + "\nAND (warpBackgroundRun.dist_group = '%s'" + " OR warpBackgroundRun.dist_group IS NULL)", + dist_group); + } + } + + if (!psDBTransaction(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); // Matching rows + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + psS64 chip_bg_id = psMetadataLookupS64(NULL, md, "chip_bg_id"); + + warpRunRow *row = warpRunObjectFromMetadata(md); + if (!row) { + psError(psErrorCodeLast(), false, "failed to convert metadata into fakeRun"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + if (!warpBackgroundRunInsert(config->dbh, 0, row->warp_id, chip_bg_id, "new", + workdir ? workdir : row->workdir, + label ? label : row->label, + data_group ? data_group : row->data_group, + dist_group ? dist_group : row->dist_group, + reduction ? reduction : row->reduction, + note ? note : row->note, + NULL, 0)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(row); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + return true; +} + +static bool updatewarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "warpBackgroundRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "warpBackgroundRun.data_group","LIKE"); + PXOPT_COPY_STR(config->args, where, "-dist_group", "warpBackgroundRun.dist_group","LIKE"); + pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "LIKE"); + PXOPT_COPY_TIME(config->args, where, "-registered_begin", "warpBackgroundRun.registered", ">="); + PXOPT_COPY_TIME(config->args, where, "-registered_end", "warpBackgroundRun.registered", "<"); + + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + if (destreaked) { + psMetadataAddS64(where, PS_LIST_TAIL, "warpBackgroundRun.magicked", PS_META_DUPLICATE_OK, ">", 0); + } + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE warpBackgroundRun"); + bool result = pxUpdateRun(config, where, &query, "warpBackgroundRun", "warp_bg_id", + "warpBackgroundSkyfile", true); + + psFree(query); + psFree(where); + + return result; +} + +static bool towarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpBackgroundRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("bgtool_towarp.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + psString whereStr = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereStr, "\n AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQueryF(config->dbh, query, whereStr)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(whereStr); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "Unable to fetch result of query %s", query); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundRun", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool warpinputsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("bgtool_warpinputs.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundSkyfile", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addwarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(warp_bg_id, config->args, "-warp_bg_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + + // optional + + PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_pswarp, config->args, "-ver_pswarp", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); + + psString ver_code = pxMergeCodeVersions(ver_pslib, ver_psmodules); + ver_code = pxMergeCodeVersions(ver_code, ver_pswarp); + ver_code = pxMergeCodeVersions(ver_code, ver_ppstats); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!warpBackgroundSkyfileInsert(config->dbh, warp_bg_id, skycell_id, path_base, magicked, dtime_script, + hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix, + maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + return true; +} + +static bool warpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpBackgroundSkyfile.skycell_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "LIKE"); + pxAddLabelSearchArgs(config, where, "-data_group", "warpBackgroundRun.data_group", "LIKE"); + pxAddLabelSearchArgs(config, where, "-dist_group", "warpBackgroundRun.data_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("bgtool_warp.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + psString magicWhere = NULL; + if (!pxmagicAddWhere(config, &magicWhere, "warpBackgroundSkyfile")) { + psError(psErrorCodeLast(), false, "pxMagicAddWhere failed"); + return false; + } + if (!pxspaceAddWhere(config, &magicWhere, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else if (!all && !magicWhere) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + if (magicWhere) { + psStringAppend(&query, "%s %s", psListLength(where->list) ? "AND" : "WHERE", magicWhere); + } + psFree(magicWhere); + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundSkyfile", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + } + psFree(output); + + return true; +} + + +static bool advancewarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warp_bg_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "label", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psString select = pxDataGet("bgtool_advancewarp.sql"); + if (!select) { + psError(psErrorCodeLast(), false, "failed to retrieve SQL statement"); + return false; + } + + psString selectWhere = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&selectWhere, "\n AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&select, " %s", limitString); + psFree(limitString); + } + + if (!psDBTransaction(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, select, selectWhere)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(select); + psFree(selectWhere); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(select); + psFree(selectWhere); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + bool status = true; // Status of MD lookup + psS64 warp_bg_id = psMetadataLookupS64(&status, row, "warp_bg_id"); + if (!status) { + psError(PXTOOLS_ERR_PROG, true, "failed to look up value for warp_bg_id"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psS64 magicked = psMetadataLookupS64(&status, row, "magicked"); + if (!status) { + psError(PXTOOLS_ERR_PROG, true, "failed to look up value for magicked"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + if (!p_psDBRunQueryF(config->dbh, + "UPDATE warpBackgroundRun " + "SET state = 'full', magicked = %" PRId64 " " + " WHERE warp_bg_id = %" PRId64, + magicked, warp_bg_id)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + psS64 numUpdated = psDBAffectedRows(config->dbh); + if (numUpdated != 1) { + psError(PXTOOLS_ERR_PROG, true, "should have affected 1 row"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + psFree(output); + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + return true; +} + +static bool revertwarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpBackgroundSkyfile.skycell_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "warpBackgroundSkyfile.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required"); + return false; + } + + psString query = pxDataGet("bgtool_revertwarp.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + int numDeleted = psDBAffectedRows(config->dbh); + psLogMsg("bgtool", PS_LOG_INFO, "Deleted %d warpBackgroundSkyfiles", numDeleted); + + return true; +} + +static bool tocleanwarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "=="); + + psString query = pxDataGet("bgtool_tocleanwarp.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundRun", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + + return true; +} + +static bool cleanedwarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + if (!pxIsValidCleanedState(state)) { + psError(PXTOOLS_ERR_CONFIG, true, "Invalid state: %s", state); + return false; + } + + if (!psListLength(where->list)) { + psError(PXTOOLS_ERR_CONFIG, true, "No search restrictions set."); + return false; + } + + psString query = pxDataGet("bgtool_cleanedwarp.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, state)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + psFree(query); + + return true; +} + +static bool exportwarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(warp_bg_id, config->args, "-warp_bg_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warp_bg_id", "=="); + + bool status = exportTables(config, outfile, warpTables, where, clean); + + psFree(where); + return status; +} + +static bool importwarpMode(pxConfig *config) +{ + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); + + return importTables(config, infile, warpTables); +} Index: branches/eam_branches/ipp-20100621/ippTools/src/bgtool.h =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/bgtool.h (revision 28794) +++ branches/eam_branches/ipp-20100621/ippTools/src/bgtool.h (revision 28794) @@ -0,0 +1,59 @@ +/* + * bgtool.h + * + * Copyright (C) 2006-2010 Joshua Hoblitt, Paul Price + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef BGTOOL_H +#define BGTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + BGTOOL_MODE_NONE = 0x0, + // Chip stage + BGTOOL_MODE_DEFINECHIP, + BGTOOL_MODE_UPDATECHIP, + BGTOOL_MODE_TOCHIP, + BGTOOL_MODE_CHIPINPUTS, + BGTOOL_MODE_ADDCHIP, + BGTOOL_MODE_CHIP, + BGTOOL_MODE_ADVANCECHIP, + BGTOOL_MODE_REVERTCHIP, + // Warp stage + BGTOOL_MODE_DEFINEWARP, + BGTOOL_MODE_UPDATEWARP, + BGTOOL_MODE_TOWARP, + BGTOOL_MODE_WARPINPUTS, + BGTOOL_MODE_ADDWARP, + BGTOOL_MODE_WARP, + BGTOOL_MODE_ADVANCEWARP, + BGTOOL_MODE_REVERTWARP, + // Cleanups + BGTOOL_MODE_TOCLEANCHIP, + BGTOOL_MODE_CLEANEDCHIP, + BGTOOL_MODE_TOCLEANWARP, + BGTOOL_MODE_CLEANEDWARP, + // Exporting + BGTOOL_MODE_EXPORTCHIP, + BGTOOL_MODE_IMPORTCHIP, + BGTOOL_MODE_EXPORTWARP, + BGTOOL_MODE_IMPORTWARP, +} bgtoolMode; + +pxConfig *bgtoolConfig(pxConfig *config, int argc, char **argv); + +#endif // BGTOOL_H Index: branches/eam_branches/ipp-20100621/ippTools/src/bgtoolConfig.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/bgtoolConfig.c (revision 28794) +++ branches/eam_branches/ipp-20100621/ippTools/src/bgtoolConfig.c (revision 28794) @@ -0,0 +1,424 @@ +/* + * bgtoolConfig.c + * + * Copyright (C) 2006-2010 Joshua Hoblitt, Paul Price + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "bgtool.h" + +pxConfig *bgtoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -definechip + psMetadata *definechipArgs = psMetadataAlloc(); + psMetadataAddS64(definechipArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); + psMetadataAddS64(definechipArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(definechipArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(definechipArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-ra_min", 0, "search by min RA (degrees) ", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-ra_max", 0, "search by max RA (degrees) ", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-decl_min", 0, "search by min DEC (degrees)", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-decl_max", 0, "search by max DEC (degrees)", NAN); + psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-exp_time_min", 0, "search by min exposure time", NAN); + psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-exp_time_max", 0, "search by max exposure time", NAN); + psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by min fraction of saturated pixels", NAN); + psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by max fraction of saturated pixels", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-bg_min", 0, "search by min background", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-bg_max", 0, "search by max background", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by min background standard deviation", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max background standard deviation", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by min background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-alt_min", 0, "search by min altitude", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-alt_max", 0, "search by max altitude", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-az_min", 0, "search by min azimuth ", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-az_max", 0, "search by max azimuth ", NAN); + psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN); + psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL); + psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search on chipRun label", NULL); + psMetadataAddBool(definechipArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false); + psMetadataAddBool(definechipArgs, PS_LIST_TAIL, "-rerun", 0, "re-run data?", false); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + + psMetadataAddTime(definechipArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definechipArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definechipArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + + // -updatechip + psMetadata *updatechipArgs = psMetadataAlloc(); + psMetadataAddS64(updatechipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); + psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label (LIKE comparison)", NULL); + psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-dist_group", 0, "search by dist_group (LIKE comparison)", NULL); + psMetadataAddTime(updatechipArgs, PS_LIST_TAIL, "-registered_begin", 0, "search by registration time (>=)", NULL); + psMetadataAddTime(updatechipArgs, PS_LIST_TAIL, "-registered_end", 0, "search by registration time (<)", NULL); + psMetadataAddBool(updatechipArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false); + psMetadataAddBool(updatechipArgs, PS_LIST_TAIL, "-pretend", 0, "only pretend to run the query", false); + psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0); + psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + + // -tochip + psMetadata *tochipArgs = psMetadataAlloc(); + psMetadataAddS64(tochipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); + psMetadataAddStr(tochipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddU64(tochipArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tochipArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -chipinputs + psMetadata *chipinputsArgs = psMetadataAlloc(); + psMetadataAddS64(chipinputsArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); + psMetadataAddStr(chipinputsArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id", 0); + psMetadataAddU64(chipinputsArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(chipinputsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addchip + psMetadata *addchipArgs = psMetadataAlloc(); + psMetadataAddS64(addchipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "define chip_bg_id (required)", 0); + psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-class_id", 0, "define class_id (required)", NULL); + + psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location (required)", NULL); + psMetadataAddS64(addchipArgs, PS_LIST_TAIL, "-set_magicked", 0, "define if this skycell has been magicked", 0); + psMetadataAddF32(addchipArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL); + psMetadataAddS16(addchipArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddS16(addchipArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL); + psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL); + psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-ver_ppbackground", 0, "define ppBackground version", NULL); + psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); + psMetadataAddF64(addchipArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addchipArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddS32(addchipArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0); + psMetadataAddF32(addchipArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN); + psMetadataAddF32(addchipArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN); + psMetadataAddF32(addchipArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN); + psMetadataAddF32(addchipArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN); + + // -chip + psMetadata *chipArgs = psMetadataAlloc(); + psMetadataAddS64(chipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); + psMetadataAddStr(chipArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id", NULL); + psMetadataAddS16(chipArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(chipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddStr(chipArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by data_group", NULL); + psMetadataAddStr(chipArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by dist_group", NULL); + pxmagicAddArguments(chipArgs); + pxspaceAddArguments(chipArgs); + psMetadataAddBool(chipArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(chipArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(chipArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -advancechip + psMetadata *advancechipArgs = psMetadataAlloc(); + psMetadataAddS64(advancechipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); + psMetadataAddStr(advancechipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label ", NULL); + psMetadataAddU64(advancechipArgs, PS_LIST_TAIL, "-limit", 0, "search limit", 0); + + // -revertchip + psMetadata *revertchipArgs = psMetadataAlloc(); + psMetadataAddS64(revertchipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); + psMetadataAddStr(revertchipArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id", NULL); + psMetadataAddStr(revertchipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddS16(revertchipArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertchipArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -tocleanchip + psMetadata *tocleanchipArgs = psMetadataAlloc(); + psMetadataAddStr(tocleanchipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddBool(tocleanchipArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(tocleanchipArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -cleanedchip + psMetadata *cleanedchipArgs = psMetadataAlloc(); + psMetadataAddS64(cleanedchipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); + psMetadataAddStr(cleanedchipArgs, PS_LIST_TAIL, "-state", 0, "cleaned state to set", NULL); + + // -exportchip + psMetadata *exportchipArgs = psMetadataAlloc(); + psMetadataAddS64(exportchipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "export this chip_bg_id (required)", 0); + psMetadataAddStr(exportchipArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddBool(exportchipArgs, PS_LIST_TAIL, "-clean", 0, "export run in cleaned state", false); + + // -importchip + psMetadata *importchipArgs = psMetadataAlloc(); + psMetadataAddStr(importchipArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + + + // -definewarp + psMetadata *definewarpArgs = psMetadataAlloc(); + psMetadataAddS64(definewarpArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", 0); + psMetadataAddS64(definewarpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(definewarpArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(definewarpArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-ra_min", 0, "search by min RA (degrees) ", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-ra_max", 0, "search by max RA (degrees) ", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-decl_min", 0, "search by min DEC (degrees)", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-decl_max", 0, "search by max DEC (degrees)", NAN); + psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-exp_time_min", 0, "search by min exposure time", NAN); + psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-exp_time_max", 0, "search by max exposure time", NAN); + psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by min fraction of saturated pixels", NAN); + psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by max fraction of saturated pixels", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-bg_min", 0, "search by min background", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-bg_max", 0, "search by max background", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by min background standard deviation", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max background standard deviation", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by min background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-alt_min", 0, "search by min altitude", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-alt_max", 0, "search by max altitude", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-az_min", 0, "search by min azimuth ", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-az_max", 0, "search by max azimuth ", NAN); + psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN); + psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL); + psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-warp_label", PS_META_DUPLICATE_OK, "search on warpRun label", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-chip_label", PS_META_DUPLICATE_OK, "search on chipBackgroundRun label", NULL); + psMetadataAddBool(definewarpArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false); + psMetadataAddBool(definewarpArgs, PS_LIST_TAIL, "-rerun", 0, "rerun data?", false); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + + psMetadataAddTime(definewarpArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definewarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definewarpArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + + // -updatewarp + psMetadata *updatewarpArgs = psMetadataAlloc(); + psMetadataAddS64(updatewarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0); + psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label (LIKE comparison)", NULL); + psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-dist_group", 0, "search by dist_group (LIKE comparison)", NULL); + psMetadataAddTime(updatewarpArgs, PS_LIST_TAIL, "-registered_begin", 0, "search by registration time (>=)", NULL); + psMetadataAddTime(updatewarpArgs, PS_LIST_TAIL, "-registered_end", 0, "search by registration time (<)", NULL); + psMetadataAddBool(updatewarpArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false); + psMetadataAddBool(updatewarpArgs, PS_LIST_TAIL, "-pretend", 0, "only pretend to run the query", false); + psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0); + psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + + // -towarp + psMetadata *towarpArgs = psMetadataAlloc(); + psMetadataAddS64(towarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0); + psMetadataAddStr(towarpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddU64(towarpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(towarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -warpinputs + psMetadata *warpinputsArgs = psMetadataAlloc(); + psMetadataAddS64(warpinputsArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0); + psMetadataAddStr(warpinputsArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", 0); + psMetadataAddU64(warpinputsArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(warpinputsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addwarp + psMetadata *addwarpArgs = psMetadataAlloc(); + psMetadataAddS64(addwarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "define warp_bg_id (required)", 0); + psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell_id (required)", NULL); + + psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); + psMetadataAddS64(addwarpArgs, PS_LIST_TAIL, "-set_magicked", 0, "define if this skycell has been magicked", 0); + psMetadataAddF32(addwarpArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddS16(addwarpArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddS16(addwarpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL); + psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL); + psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-ver_pswarp", 0, "define pswarp version", NULL); + psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); + psMetadataAddF64(addwarpArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addwarpArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddS32(addwarpArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0); + psMetadataAddF32(addwarpArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN); + psMetadataAddF32(addwarpArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN); + psMetadataAddF32(addwarpArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN); + psMetadataAddF32(addwarpArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN); + + // -warp + psMetadata *warpArgs = psMetadataAlloc(); + psMetadataAddS64(warpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0); + psMetadataAddStr(warpArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", NULL); + psMetadataAddS16(warpArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(warpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddStr(warpArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by data_group", NULL); + psMetadataAddStr(warpArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by dist_group", NULL); + pxmagicAddArguments(warpArgs); + pxspaceAddArguments(warpArgs); + psMetadataAddBool(warpArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(warpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(warpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -advancewarp + psMetadata *advancewarpArgs = psMetadataAlloc(); + psMetadataAddS64(advancewarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0); + psMetadataAddStr(advancewarpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label ", NULL); + psMetadataAddU64(advancewarpArgs, PS_LIST_TAIL, "-limit", 0, "search limit", 0); + + // -revertwarp + psMetadata *revertwarpArgs = psMetadataAlloc(); + psMetadataAddS64(revertwarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0); + psMetadataAddStr(revertwarpArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", NULL); + psMetadataAddStr(revertwarpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddS16(revertwarpArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertwarpArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -tocleanwarp + psMetadata *tocleanwarpArgs = psMetadataAlloc(); + psMetadataAddStr(tocleanwarpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddBool(tocleanwarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(tocleanwarpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -cleanedwarp + psMetadata *cleanedwarpArgs = psMetadataAlloc(); + psMetadataAddS64(cleanedwarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0); + psMetadataAddStr(cleanedwarpArgs, PS_LIST_TAIL, "-state", 0, "cleaned state to set", NULL); + + // -exportwarp + psMetadata *exportwarpArgs = psMetadataAlloc(); + psMetadataAddS64(exportwarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "export this warp_bg_id (required)", 0); + psMetadataAddStr(exportwarpArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddBool(exportwarpArgs, PS_LIST_TAIL, "-clean", 0, "export run in cleaned state", false); + + // -importwarp + psMetadata *importwarpArgs = psMetadataAlloc(); + psMetadataAddStr(importwarpArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + + + psFree(now); + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definechip", "", BGTOOL_MODE_DEFINECHIP, definechipArgs); + PXOPT_ADD_MODE("-updatechip", "", BGTOOL_MODE_UPDATECHIP, updatechipArgs); + PXOPT_ADD_MODE("-tochip", "", BGTOOL_MODE_TOCHIP, tochipArgs); + PXOPT_ADD_MODE("-chipinputs", "", BGTOOL_MODE_CHIPINPUTS, chipinputsArgs); + PXOPT_ADD_MODE("-addchip", "", BGTOOL_MODE_ADDCHIP, addchipArgs); + PXOPT_ADD_MODE("-chip", "", BGTOOL_MODE_CHIP, chipArgs); + PXOPT_ADD_MODE("-advancechip", "", BGTOOL_MODE_ADVANCECHIP, advancechipArgs); + PXOPT_ADD_MODE("-revertchip", "", BGTOOL_MODE_REVERTCHIP, revertchipArgs); + PXOPT_ADD_MODE("-tocleanchip", "", BGTOOL_MODE_TOCLEANCHIP, tocleanchipArgs); + PXOPT_ADD_MODE("-cleanedchip", "", BGTOOL_MODE_CLEANEDCHIP, cleanedchipArgs); + PXOPT_ADD_MODE("-exportchip", "", BGTOOL_MODE_EXPORTCHIP, exportchipArgs); + PXOPT_ADD_MODE("-importchip", "", BGTOOL_MODE_IMPORTCHIP, importchipArgs); + + PXOPT_ADD_MODE("-definewarp", "", BGTOOL_MODE_DEFINEWARP, definewarpArgs); + PXOPT_ADD_MODE("-updatewarp", "", BGTOOL_MODE_UPDATEWARP, updatewarpArgs); + PXOPT_ADD_MODE("-towarp", "", BGTOOL_MODE_TOWARP, towarpArgs); + PXOPT_ADD_MODE("-warpinputs", "", BGTOOL_MODE_WARPINPUTS, warpinputsArgs); + PXOPT_ADD_MODE("-addwarp", "", BGTOOL_MODE_ADDWARP, addwarpArgs); + PXOPT_ADD_MODE("-warp", "", BGTOOL_MODE_WARP, warpArgs); + PXOPT_ADD_MODE("-advancewarp", "", BGTOOL_MODE_ADVANCEWARP, advancewarpArgs); + PXOPT_ADD_MODE("-revertwarp", "", BGTOOL_MODE_REVERTWARP, revertwarpArgs); + PXOPT_ADD_MODE("-tocleanwarp", "", BGTOOL_MODE_TOCLEANWARP, tocleanwarpArgs); + PXOPT_ADD_MODE("-cleanedwarp", "", BGTOOL_MODE_CLEANEDWARP, cleanedwarpArgs); + PXOPT_ADD_MODE("-exportwarp", "", BGTOOL_MODE_EXPORTWARP, exportwarpArgs); + PXOPT_ADD_MODE("-importwarp", "", BGTOOL_MODE_IMPORTWARP, importwarpArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: branches/eam_branches/ipp-20100621/ippTools/src/chiptool.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/chiptool.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/chiptool.c (revision 28794) @@ -335,4 +335,5 @@ pxchipGetSearchArgs (config, where); // rawExp only pxAddLabelSearchArgs (config, where, "-label", "chipOld.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "chipOld.data_group", "LIKE"); // psListLength(where->list) is at least 1 because exp_type defaults to "object" Index: branches/eam_branches/ipp-20100621/ippTools/src/chiptoolConfig.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/chiptoolConfig.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/chiptoolConfig.c (revision 28794) @@ -67,5 +67,6 @@ psMetadata *definecopyArgs = psMetadataAlloc(); pxchipSetSearchArgs (definecopyArgs); - psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by rawExp label (LIKE comparison)", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL); psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-exp_type", PS_META_REPLACE, "search by exp_type", "object"); Index: branches/eam_branches/ipp-20100621/ippTools/src/dettool.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/dettool.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/dettool.c (revision 28794) @@ -255,6 +255,6 @@ // get -ref_det_id and -ref_iter : required for 'verify' mode / disallowed otherwise - PXOPT_LOOKUP_S64(ref_det_id, config->args, "-ref_det_id", true, false); - PXOPT_LOOKUP_S32(ref_iter, config->args, "-ref_iter", true, false); + PXOPT_LOOKUP_S64(ref_det_id, config->args, "-ref_det_id", false, false); + PXOPT_LOOKUP_S32(ref_iter, config->args, "-ref_iter", false, false); if (!strcmp(mode, "verify") && ((ref_det_id == 0) || (ref_iter == -1))) { psError(PS_ERR_UNKNOWN, false, "verify mode requires both -ref_det_id and -ref_iter"); Index: branches/eam_branches/ipp-20100621/ippTools/src/dettoolConfig.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/dettoolConfig.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/dettoolConfig.c (revision 28794) @@ -299,9 +299,9 @@ // -revertprocessedimfile psMetadata *revertprocessedimfileArgs = psMetadataAlloc(); - psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID (required)", 0); + psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); psMetadataAddS16(revertprocessedimfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); - + psMetadataAddBool(revertprocessedimfileArgs, PS_LIST_TAIL, "-all-run", 0, "revert all in state 'run'", false); // -updateprocessedimfile psMetadata *updateprocessedimfileArgs = psMetadataAlloc(); @@ -361,8 +361,8 @@ // -revertprocessedexp psMetadata *revertprocessedexpArgs = psMetadataAlloc(); - psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID", 0); psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); - + psMetadataAddBool(revertprocessedexpArgs, PS_LIST_TAIL, "-all-run", 0, "revert all in state 'run'", false); // -updateprocessedexp psMetadata *updateprocessedexpArgs = psMetadataAlloc(); @@ -425,8 +425,9 @@ // -revertstacked psMetadata *revertstackedArgs= psMetadataAlloc(); - psMetadataAddS64(revertstackedArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID (required)", 0); + psMetadataAddS64(revertstackedArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); psMetadataAddS32(revertstackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(revertstackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); psMetadataAddS16(revertstackedArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertstackedArgs, PS_LIST_TAIL, "-all-run", 0, "revert all in state 'run'", false); // -updatestacked @@ -477,9 +478,9 @@ // -revertnormalizedstat psMetadata *revertnormalizedstatArgs= psMetadataAlloc(); - psMetadataAddS64(revertnormalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS64(revertnormalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID", 0); psMetadataAddS32(revertnormalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(revertnormalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); psMetadataAddS16(revertnormalizedstatArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); - + psMetadataAddBool(revertnormalizedstatArgs, PS_LIST_TAIL, "-all-run", 0, "revert all in state 'run'", false); // -updatenormalizedstat psMetadata *updatenormalizedstatArgs = psMetadataAlloc(); @@ -539,9 +540,9 @@ // -revertnormalizedimfile psMetadata *revertnormalizedimfileArgs = psMetadataAlloc(); - psMetadataAddS64(revertnormalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS64(revertnormalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID", 0); psMetadataAddS32(revertnormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(revertnormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); psMetadataAddS16(revertnormalizedimfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); - + psMetadataAddBool(revertnormalizedimfileArgs, PS_LIST_TAIL, "-all-run", 0, "revert all in state 'run'", false); // -updatenormalizedimfile psMetadata *updatenormalizedimfileArgs = psMetadataAlloc(); @@ -599,7 +600,8 @@ // -revertnormalizedexp psMetadata *revertnormalizedexpArgs = psMetadataAlloc(); - psMetadataAddS64(revertnormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS64(revertnormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID", 0); psMetadataAddS32(revertnormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddS16(revertnormalizedexpArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertnormalizedexpArgs, PS_LIST_TAIL, "-all-run", 0, "revert all in state 'run'", false); // -updatenormalizedexp @@ -675,9 +677,10 @@ // -revertresidimfile psMetadata *revertresidimfileArgs = psMetadataAlloc(); - psMetadataAddS64(revertresidimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS64(revertresidimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID", 0); psMetadataAddS32(revertresidimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddS64(revertresidimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by detrend ID", 0); psMetadataAddStr(revertresidimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID", NULL); psMetadataAddS16(revertresidimfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertresidimfileArgs, PS_LIST_TAIL, "-all-run", 0, "revert all in state 'run'", false); // -updateresidimfile @@ -752,9 +755,9 @@ // -revertresidexp psMetadata *revertresidexpArgs = psMetadataAlloc(); - psMetadataAddS64(revertresidexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS64(revertresidexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID", 0); psMetadataAddS32(revertresidexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddS64(revertresidexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by detrend ID", 0); psMetadataAddS16(revertresidexpArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); - + psMetadataAddBool(revertresidexpArgs, PS_LIST_TAIL, "-all-run", 0, "revert all in state 'run'", false); // -updateresidexp psMetadata *updateresidexpArgs = psMetadataAlloc(); @@ -819,7 +822,8 @@ // -revertdetrunsummary psMetadata *revertdetrunsummaryArgs = psMetadataAlloc(); - psMetadataAddS64(revertdetrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS64(revertdetrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID", 0); psMetadataAddS32(revertdetrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddS16(revertdetrunsummaryArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertdetrunsummaryArgs, PS_LIST_TAIL, "-all-run", 0, "revert all in state 'run'", false); // -updatedetrunsummary Index: branches/eam_branches/ipp-20100621/ippTools/src/dettool_detrunsummary.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/dettool_detrunsummary.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/dettool_detrunsummary.c (revision 28794) @@ -306,5 +306,11 @@ PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); - PXOPT_COPY_STR(config->args, where, "-fault", "fault", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all-run")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } psString query = pxDataGet("dettool_revertdetrunsummary.sql"); @@ -327,9 +333,4 @@ } psFree(query); - - if (psDBAffectedRows(config->dbh) < 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); - return false; - } return true; Index: branches/eam_branches/ipp-20100621/ippTools/src/dettool_normalizedexp.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/dettool_normalizedexp.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/dettool_normalizedexp.c (revision 28794) @@ -222,4 +222,10 @@ PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all-run")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } psString query = pxDataGet("dettool_revertnormalizedexp.sql"); @@ -242,9 +248,4 @@ } psFree(query); - - if (psDBAffectedRows(config->dbh) < 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); - return false; - } return true; Index: branches/eam_branches/ipp-20100621/ippTools/src/dettool_normalizedimfile.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/dettool_normalizedimfile.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/dettool_normalizedimfile.c (revision 28794) @@ -202,4 +202,10 @@ PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all-run")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } psString query = pxDataGet("dettool_revertnormalizedimfile.sql"); @@ -222,9 +228,4 @@ } psFree(query); - - if (psDBAffectedRows(config->dbh) < 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); - return false; - } return true; Index: branches/eam_branches/ipp-20100621/ippTools/src/dettool_normalizedstat.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/dettool_normalizedstat.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/dettool_normalizedstat.c (revision 28794) @@ -182,4 +182,10 @@ PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all-run")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } psString query = pxDataGet("dettool_revertnormalizedstat.sql"); @@ -202,9 +208,4 @@ } psFree(query); - - if (psDBAffectedRows(config->dbh) < 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); - return false; - } return true; Index: branches/eam_branches/ipp-20100621/ippTools/src/dettool_processedexp.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/dettool_processedexp.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/dettool_processedexp.c (revision 28794) @@ -232,4 +232,10 @@ PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all-run")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } psString query = pxDataGet("dettool_revertprocessedexp.sql"); @@ -252,9 +258,4 @@ } psFree(query); - - if (psDBAffectedRows(config->dbh) < 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); - return false; - } return true; Index: branches/eam_branches/ipp-20100621/ippTools/src/dettool_processedimfile.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/dettool_processedimfile.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/dettool_processedimfile.c (revision 28794) @@ -264,4 +264,10 @@ PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all-run")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } psString query = pxDataGet("dettool_revertprocessedimfile.sql"); @@ -284,9 +290,4 @@ } psFree(query); - - if (psDBAffectedRows(config->dbh) < 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); - return false; - } return true; Index: branches/eam_branches/ipp-20100621/ippTools/src/dettool_residexp.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/dettool_residexp.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/dettool_residexp.c (revision 28794) @@ -275,4 +275,10 @@ PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all-run")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psString query = pxDataGet("dettool_revertresidexp.sql"); if (!query) { @@ -294,9 +300,4 @@ } psFree(query); - - if (psDBAffectedRows(config->dbh) < 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); - return false; - } return true; Index: branches/eam_branches/ipp-20100621/ippTools/src/dettool_residimfile.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/dettool_residimfile.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/dettool_residimfile.c (revision 28794) @@ -250,4 +250,10 @@ PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all-run")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } psString query = pxDataGet("dettool_revertresidimfile.sql"); @@ -270,9 +276,4 @@ } psFree(query); - - if (psDBAffectedRows(config->dbh) < 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); - return false; - } return true; Index: branches/eam_branches/ipp-20100621/ippTools/src/dettool_stack.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/dettool_stack.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/dettool_stack.c (revision 28794) @@ -287,4 +287,10 @@ PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all-run")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } psString query = pxDataGet("dettool_revertstacked.sql"); @@ -307,9 +313,4 @@ } psFree(query); - - if (psDBAffectedRows(config->dbh) < 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); - return false; - } return true; Index: branches/eam_branches/ipp-20100621/ippTools/src/diffphottool.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/diffphottool.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/diffphottool.c (revision 28794) @@ -177,5 +177,5 @@ set_reduction ? set_reduction : reduction, registered, - set_note ? set_note : note); + set_note ? set_note : note, 0); if (!diffPhotRunInsertObject(config->dbh, run)) { psError(psErrorCodeLast(), false, "database error"); @@ -430,7 +430,9 @@ psMetadata *row = output->data[i]; // Row of interest psS64 diff_phot_id = psMetadataLookupS64(NULL, row, "diff_phot_id"); - - const char *query = "UPDATE diffPhotRun SET state = 'full' WHERE diff_phot_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, diff_phot_id)) { + psS64 magicked = psMetadataLookupS64(NULL, row, "magicked"); + + const char *query = "UPDATE diffPhotRun SET state = 'full', magicked = %" PRId64 + " WHERE diff_phot_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, magicked, diff_phot_id)) { psError(psErrorCodeLast(), false, "failed to change state for diff_phot_id %" PRId64, diff_phot_id); Index: branches/eam_branches/ipp-20100621/ippTools/src/disttool.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/disttool.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/disttool.c (revision 28794) @@ -210,4 +210,20 @@ psStringAppend(&query, " AND (chipRun.dist_group = '%s')", dist_group); } + } else if (!strcmp(stage, "chip_bg")) { + magicRunType = "chipBackgroundRun"; + runJoinStr = "chipBackgroundRun.chip_bg_id"; + query = pxDataGet("disttool_definebyquery_chip_bg.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (label) { + psStringAppend(&query, " AND (chipBackgroundRun.label = '%s')", label); + } + if (dist_group) { + psStringAppend(&query, " AND (chipBackgroundRun.dist_group = '%s')", dist_group); + } } else if (!strcmp(stage, "camera")) { magicRunType = "camRun"; // This is used below to set the magicked business @@ -258,4 +274,21 @@ if (dist_group) { psStringAppend(&query, " AND (warpRun.dist_group = '%s')", dist_group); + } + + } else if (!strcmp(stage, "warp_bg")) { + magicRunType = "warpBackgroundRun"; + runJoinStr = "warpBackgroundRun.warp_bg_id"; + query = pxDataGet("disttool_definebyquery_warp_bg.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (label) { + psStringAppend(&query, " AND (warpBackgroundRun.label = '%s')", label); + } + if (dist_group) { + psStringAppend(&query, " AND (warpBackgroundRun.dist_group = '%s')", dist_group); } Index: branches/eam_branches/ipp-20100621/ippTools/src/magicdstool.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/magicdstool.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/magicdstool.c (revision 28794) @@ -122,6 +122,8 @@ PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chip_bg_id", "=="); PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warp_bg_id", "=="); PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); PXOPT_COPY_S64(config->args, where, "-magic_id","magicRun.magic_id", "=="); @@ -140,4 +142,7 @@ query = pxDataGet("magicdstool_definebyquery_chip.sql"); break; + case IPP_STAGE_CHIP_BG: + query = pxDataGet("magicdstool_definebyquery_chip_bg.sql"); + break; case IPP_STAGE_CAMERA: query = pxDataGet("magicdstool_definebyquery_camera.sql"); @@ -145,4 +150,7 @@ case IPP_STAGE_WARP: query = pxDataGet("magicdstool_definebyquery_warp.sql"); + break; + case IPP_STAGE_WARP_BG: + query = pxDataGet("magicdstool_definebyquery_warp_bg.sql"); break; case IPP_STAGE_DIFF: @@ -720,4 +728,7 @@ query = "UPDATE chipProcessedImfile SET magicked = %" PRId64 " where chip_id = %" PRId64 " AND class_id = '%s'"; break; + case IPP_STAGE_CHIP_BG: + query = "UPDATE chipBackgroundImfile SET magicked = %" PRId64 " where chip_bg_id = %" PRId64 " AND class_id = '%s'"; + break; case IPP_STAGE_CAMERA: // no there is no magicked column in camProcessedExp so we have nothing to do @@ -727,4 +738,7 @@ query = "UPDATE warpSkyfile SET magicked = %" PRId64 " where warp_id = %" PRId64 " AND skycell_id = '%s'"; break; + case IPP_STAGE_WARP_BG: + query = "UPDATE warpBackgroundSkyfile SET magicked = %" PRId64 " where warp_bg_id = %" PRId64 " AND skycell_id = '%s'"; + break; case IPP_STAGE_DIFF: query = "UPDATE diffSkyfile SET magicked = %" PRId64 " where diff_id = %" PRId64 " AND skycell_id = '%s'"; @@ -794,4 +808,7 @@ query = "UPDATE chipRun SET magicked = %" PRId64 " where chip_id = %" PRId64; break; + case IPP_STAGE_CHIP_BG: + query = "UPDATE chipBackgroundRun SET magicked = %" PRId64 " where chip_bg_id = %" PRId64; + break; case IPP_STAGE_CAMERA: query = "UPDATE camRun SET magicked = %" PRId64 " where cam_id = %" PRId64; @@ -799,4 +816,7 @@ case IPP_STAGE_WARP: query = "UPDATE warpRun SET magicked = %" PRId64 " where warp_id = %" PRId64; + break; + case IPP_STAGE_WARP_BG: + query = "UPDATE warpBackgroundRun SET magicked = %" PRId64 " where warp_bg_id = %" PRId64; break; case IPP_STAGE_DIFF: Index: branches/eam_branches/ipp-20100621/ippTools/src/magicdstoolConfig.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/magicdstoolConfig.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/magicdstoolConfig.c (revision 28794) @@ -60,6 +60,8 @@ psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0); psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff_id", 0); psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic_id", 0); Index: branches/eam_branches/ipp-20100621/ippTools/src/magictool.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/magictool.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/magictool.c (revision 28794) @@ -246,4 +246,5 @@ psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); // difference identifier bool inverse = psMetadataLookupU64(NULL, row, "inverse"); // Inverse subtraction? Note types! + psString diff_data_group = psMetadataLookupStr(NULL, row, "diff_data_group"); // create a new magicRun for this group @@ -256,5 +257,5 @@ "dirty", // workdir_state label, - data_group ? data_group : label, + data_group ? data_group : (diff_data_group ? diff_data_group : label), dvodb, registered, @@ -521,4 +522,6 @@ psFree(where); + psStringAppend(&query, "\nORDER BY priority DESC, magic_id"); + // treat limit == 0 as "no limit" if (limit) { @@ -830,4 +833,6 @@ psFree(where); + psStringAppend(&query, "\nORDER BY priority DESC, magic_id"); + // treat limit == 0 as "no limit" if (limit) { @@ -1365,8 +1370,8 @@ // check that state is a valid string value if (!( - (strncmp(state, "new", 4) == 0) - || (strncmp(state, "full", 5) == 0) - || (strncmp(state, "drop", 5) == 0) - || (strncmp(state, "reg", 4) == 0) + (strncmp(state, "new", 3) == 0) + || (strncmp(state, "full", 4) == 0) + || (strncmp(state, "drop", 4) == 0) + || (strncmp(state, "reg", 3) == 0) ) ) { Index: branches/eam_branches/ipp-20100621/ippTools/src/pstamptool.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/pstamptool.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/pstamptool.c (revision 28794) @@ -206,4 +206,5 @@ PXOPT_LOOKUP_S64(ds_id, config->args, "-ds_id", true, false); PXOPT_LOOKUP_STR(lastFileset, config->args, "-set_last_fileset", false, false); + PXOPT_LOOKUP_STR(uri, config->args, "-set_uri", false, false); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); @@ -211,5 +212,5 @@ PXOPT_LOOKUP_BOOL(update_timestamp, config->args, "-update_timestamp", false); - if (!state && !lastFileset && !pollInterval && !update_timestamp && !label) { + if (!state && !lastFileset && !pollInterval && !update_timestamp && !label &&!uri) { psError(PS_ERR_UNKNOWN, true, "at least one of -last_fileset or -set_state is required"); return false; @@ -222,4 +223,7 @@ } + if (uri) { + psStringAppend(&query, ", uri = '%s'", uri); + } if (state) { psStringAppend(&query, ", state = '%s'", state); @@ -738,4 +742,5 @@ PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); PXOPT_COPY_S64(config->args, where, "-fault", "fault", "=="); @@ -891,4 +896,5 @@ PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); + PXOPT_COPY_S32(config->args, where, "-fault", "pstampJob.fault", "=="); PXOPT_COPY_STR(config->args, where, "-state", "pstampJob.state", "=="); Index: branches/eam_branches/ipp-20100621/ippTools/src/pstamptoolConfig.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/pstamptoolConfig.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/pstamptoolConfig.c (revision 28794) @@ -62,4 +62,5 @@ psMetadataAddS64(moddatastoreArgs, PS_LIST_TAIL, "-ds_id", 0, "define ds_id", 0); psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-set_last_fileset", 0, "set last_fileset seen", NULL); + psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-set_uri", 0, "set uri for data store", NULL); psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); @@ -146,4 +147,5 @@ psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-req_id", 0, "select by request ID", 0); psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-job_id", 0, "select by job ID", 0); + psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-dep_id", 0, "select by dependent ID", 0); psMetadataAddS16(listjobArgs, PS_LIST_TAIL, "-fault", 0, "select by fault", 0); psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -163,5 +165,6 @@ psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-job_id", 0, "job_id of jobs to update", 0); psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-dep_id", 0, "dep_id of jobs to update", 0); - psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0, "current state of jobs to update", 0); + psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0, "current state of jobs to update", 0); + psMetadataAddS16(updatejobArgs, PS_LIST_TAIL, "-fault", 0, "current value for job fault", 0); psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-set_state", 0, "new state", NULL); psMetadataAddS16(updatejobArgs, PS_LIST_TAIL, "-set_fault", 0, "new result", 0); Index: branches/eam_branches/ipp-20100621/ippTools/src/pubtool.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/pubtool.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/pubtool.c (revision 28794) @@ -175,6 +175,6 @@ PXOPT_COPY_S64(config->args, diffphotWhere, "-client_id", "client_id", "=="); - pxAddLabelSearchArgs(config, diffphotWhere, "-label", "diffphotRun.label", "=="); - pxAddLabelSearchArgs(config, diffphotWhere, "-data_group", "diffphotRun.data_group", "LIKE"); + pxAddLabelSearchArgs(config, diffphotWhere, "-label", "diffPhotRun.label", "=="); + pxAddLabelSearchArgs(config, diffphotWhere, "-data_group", "diffPhotRun.data_group", "LIKE"); PXOPT_COPY_TIME(config->args, diffphotWhere, "-dateobs_begin", "rawExp.dateobs", ">="); PXOPT_COPY_TIME(config->args, diffphotWhere, "-dateobs_end", "rawExp.dateobs", "<="); Index: branches/eam_branches/ipp-20100621/ippTools/src/pxtools.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/pxtools.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/pxtools.c (revision 28794) @@ -48,4 +48,14 @@ } +bool pxIsValidCleanedState(const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + if (!strcmp(state, "cleaned")) return true; + if (!strcmp(state, "purged")) return true; + if (!strcmp(state, "scrubbed")) return true; + return false; +} + psString pxMergeCodeVersions(psString version1, psString version2) { @@ -211,9 +221,9 @@ // change the value for tableName.columName from 'full' to 'cleaned' if necessary -bool pxSetStateCleaned(const psString tableName, const psString columnName, psArray *rows) +bool pxSetStateCleaned(const char *tableName, const char *columnName, psArray *rows) { for (long i = 0; i < psArrayLength(rows); i++) { psMetadata *row = rows->data[i]; - psString state = psMetadataLookupStr(NULL, row, columnName); + const char *state = psMetadataLookupStr(NULL, row, columnName); if (!state) { psError(PS_ERR_PROGRAMMING, false, "%s not found in row %ld of table %s", Index: branches/eam_branches/ipp-20100621/ippTools/src/pxtools.h =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/pxtools.h (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/pxtools.h (revision 28794) @@ -53,20 +53,21 @@ bool pxIsValidState(const char *state); +bool pxIsValidCleanedState(const char *state); psString pxMergeCodeVersions(psString version1, psString version2); bool pxCoalesceRunStatus(pxConfig *config, const psString dbQFile, psS64 stage_id, psString *software_ver, - psS64 *maskfrac_npix, psF32 *maskfrac_static, psF32 *maskfrac_dynamic, - psF32 *maskfrac_magic, psF32 *maskfrac_advisory); + psS64 *maskfrac_npix, psF32 *maskfrac_static, psF32 *maskfrac_dynamic, + psF32 *maskfrac_magic, psF32 *maskfrac_advisory); bool pxSetRunSoftware(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, - psString software_ver); + psString software_ver); bool pxSetRunMaskfrac(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, - psS64 maskfrac_npix, psF32 maskfrac_static, psF32 maskfrac_dynamic, - psF32 maskfrac_magic, psF32 maskfrac_advisory); + psS64 maskfrac_npix, psF32 maskfrac_static, psF32 maskfrac_dynamic, + psF32 maskfrac_magic, psF32 maskfrac_advisory); bool pxCamSetRunMaskfrac(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, - psS64 maskfrac_ref_npix, psF32 maskfrac_ref_static, psF32 maskfrac_ref_dynamic, - psF32 maskfrac_ref_magic, psF32 maskfrac_ref_advisory, - psS64 maskfrac_max_npix, psF32 maskfrac_max_static, psF32 maskfrac_max_dynamic, - psF32 maskfrac_max_magic, psF32 maskfrac_max_advisory); - -bool pxSetStateCleaned(const psString tableName, const psString columnName, psArray *rows); + psS64 maskfrac_ref_npix, psF32 maskfrac_ref_static, psF32 maskfrac_ref_dynamic, + psF32 maskfrac_ref_magic, psF32 maskfrac_ref_advisory, + psS64 maskfrac_max_npix, psF32 maskfrac_max_static, psF32 maskfrac_max_dynamic, + psF32 maskfrac_max_magic, psF32 maskfrac_max_advisory); + +bool pxSetStateCleaned(const char *tableName, const char *columnName, psArray *rows); bool pxAddLabelSearchArgs (pxConfig *config, psMetadata *where, char *field, char *name, char *op); Index: branches/eam_branches/ipp-20100621/ippTools/src/stacktool.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/stacktool.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/stacktool.c (revision 28794) @@ -78,8 +78,8 @@ MODECASE(STACKTOOL_MODE_ADDSUMSKYFILE, addsumskyfileMode); MODECASE(STACKTOOL_MODE_SUMSKYFILE, sumskyfileMode); - MODECASE(STACKTOOL_MODE_SASSSKYFILE, sassskyfileMode); + MODECASE(STACKTOOL_MODE_SASSSKYFILE, sassskyfileMode); MODECASE(STACKTOOL_MODE_REVERTSUMSKYFILE, revertsumskyfileMode); - MODECASE(STACKTOOL_MODE_TOSUMMARY, tosummaryMode); - MODECASE(STACKTOOL_MODE_ADDSUMMARY, addsummaryMode); + MODECASE(STACKTOOL_MODE_TOSUMMARY, tosummaryMode); + MODECASE(STACKTOOL_MODE_ADDSUMMARY, addsummaryMode); MODECASE(STACKTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); MODECASE(STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode); @@ -155,8 +155,8 @@ printf("%" PRId64 " %s %s %s %s\n",psMetadataLookupS64(NULL,row,"sass_id"), - psMetadataLookupStr(NULL,row,"data_group"), - psMetadataLookupStr(NULL,row,"tess_id"), - psMetadataLookupStr(NULL,row,"filter"), - psMetadataLookupStr(NULL,row,"projection_cell")); + psMetadataLookupStr(NULL,row,"data_group"), + psMetadataLookupStr(NULL,row,"tess_id"), + psMetadataLookupStr(NULL,row,"filter"), + psMetadataLookupStr(NULL,row,"projection_cell")); if (psMetadataLookupS64(NULL,row,"sass_id") == PS_MAX_S64) { @@ -170,12 +170,12 @@ psMetadataAddStr(outrow,PS_LIST_TAIL,"filter",PS_META_REPLACE,"",psMetadataLookupStr(NULL,row,"filter")); psMetadataAddStr(outrow,PS_LIST_TAIL,"projection_cell",PS_META_REPLACE,"", - psMetadataLookupStr(NULL,row,"projection_cell")); + psMetadataLookupStr(NULL,row,"projection_cell")); } printf("%" PRId64 " %s %s %s %s\n",psMetadataLookupS64(NULL,outrow,"sass_id"), - psMetadataLookupStr(NULL,outrow,"data_group"), - psMetadataLookupStr(NULL,outrow,"tess_id"), - psMetadataLookupStr(NULL,outrow,"filter"), - psMetadataLookupStr(NULL,outrow,"projection_cell")); - + psMetadataLookupStr(NULL,outrow,"data_group"), + psMetadataLookupStr(NULL,outrow,"tess_id"), + psMetadataLookupStr(NULL,outrow,"filter"), + psMetadataLookupStr(NULL,outrow,"projection_cell")); + psFree(output); @@ -184,8 +184,8 @@ return(sassRow); } - - - - + + + + @@ -239,4 +239,5 @@ PXOPT_COPY_F32(config->args, where, "-select_zpt_obs_min", "camProcessedExp.zpt_obs", ">="); PXOPT_COPY_F32(config->args, where, "-select_zpt_obs_max", "camProcessedExp.zpt_obs", "<="); + PXOPT_COPY_F32(config->args, where, "-select_astrom", "sqrt(POWER(camProcessedExp.sigma_ra, 2) + POWER(camProcessedExp.sigma_dec, 2))", "<="); PXOPT_COPY_STR(config->args, where, "-select_exp_type", "rawExp.exp_type", "=="); @@ -483,48 +484,48 @@ psFree(run); - //CZW Add an association entry here. - // Define the requested association, and insert it if it doesn't already exist - stackAssociationRow *association = pxStackAssociationDefine(config,stack_id); - psS64 sass_id; - if (!association->sass_id) { - psTrace("stacktool.association",2,"No required Association found. Adding."); - - if (!stackAssociationInsertObject(config->dbh,association)) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(output); - psFree(run); - psFree(insert); - psFree(list); - psFree(association); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return(false); - } - sass_id = psDBLastInsertID(config->dbh); - association->sass_id = sass_id; - } - // Insert the map entry for this row. - stackAssociationMapRow *maprow = stackAssociationMapRowAlloc(sass_id,stack_id); - if (!stackAssociationMapInsertObject(config->dbh,maprow)) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(output); - psFree(run); - psFree(insert); - psFree(list); - psFree(association); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return(false); - } - - + //CZW Add an association entry here. + // Define the requested association, and insert it if it doesn't already exist + stackAssociationRow *association = pxStackAssociationDefine(config,stack_id); + psS64 sass_id; + if (!association->sass_id) { + psTrace("stacktool.association",2,"No required Association found. Adding."); + + if (!stackAssociationInsertObject(config->dbh,association)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + psFree(run); + psFree(insert); + psFree(list); + psFree(association); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + sass_id = psDBLastInsertID(config->dbh); + association->sass_id = sass_id; + } + // Insert the map entry for this row. + stackAssociationMapRow *maprow = stackAssociationMapRowAlloc(sass_id,stack_id); + if (!stackAssociationMapInsertObject(config->dbh,maprow)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + psFree(run); + psFree(insert); + psFree(list); + psFree(association); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + + // Create a suitable insertion query for this run psString thisInsert = psStringCopy(insert); @@ -685,5 +686,5 @@ //CZW Add an association entry here. - + // insert the stackInputSkyfile rows psListIterator *iter = psListIteratorAlloc(warp_ids->data.list, 0, false); @@ -740,5 +741,5 @@ PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-state", "stackRun.state", "=="); - PXOPT_COPY_STR(config->args, where, "-sass_id", "stackAssociationMap.sass_id", "=="); + PXOPT_COPY_S64(config->args, where, "-sass_id", "stackAssociationMap.sass_id", "=="); if (!psListLength(where->list)) { psFree(where); @@ -1348,5 +1349,5 @@ static bool tosummaryMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, NULL); - + psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.warp_id", "=="); @@ -1357,10 +1358,10 @@ pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "LIKE"); pxAddLabelSearchArgs (config, where, "-data_group", "stackRun.data_group", "LIKE"); - + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - + // find all rawImfiles matching the default query psString query = pxDataGet("stacktool_tosummary.sql"); @@ -1380,5 +1381,5 @@ return false; } - + psFree(where); @@ -1408,5 +1409,5 @@ psError(PXTOOLS_ERR_PROG, false, "unknown error"); } - + return false; } @@ -1416,5 +1417,5 @@ return true; } - + if (psArrayLength(output)) { // negative simple so the default is true @@ -1425,5 +1426,5 @@ } } - + psFree(output); return(true); @@ -1447,5 +1448,5 @@ } psS64 numUpdated = psDBAffectedRows(config->dbh); - + if (numUpdated != 1) { psError(PS_ERR_UNKNOWN, false, "should have affected 1 row"); @@ -1453,9 +1454,9 @@ return false; } - + psFree(query); // Print anything here? - + return(true); } @@ -1471,5 +1472,5 @@ PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); PXOPT_COPY_S64(config->args, where, "-sass_id", "stackAssociationMap.sass_id", "=="); - + pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "=="); Index: branches/eam_branches/ipp-20100621/ippTools/src/stacktoolConfig.c =================================================================== --- branches/eam_branches/ipp-20100621/ippTools/src/stacktoolConfig.c (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTools/src/stacktoolConfig.c (revision 28794) @@ -90,4 +90,5 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_zpt_obs_min", 0, "define min zero point", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_zpt_obs_max", 0, "define max zero point", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_astrom", 0, "define max astrometry rms", NAN); psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random", 0, "use this number of random elements", 0); psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-min_num", 0, "minimum number of inputs", 0); @@ -115,4 +116,5 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-sass_id", 0, "search by stack association ID", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); @@ -237,6 +239,6 @@ psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL, "-sass_id", 0, "set stack Association ID", 0); psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-projection_cell", 0, "set projection cell", NULL); - psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0, "set summary path base", NULL); - + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0, "set summary path base", NULL); + // -pendingcleanuprun psMetadata *pendingcleanuprunArgs = psMetadataAlloc();