Index: trunk/ippTools/src/chiptool.c =================================================================== --- trunk/ippTools/src/chiptool.c (revision 17226) +++ trunk/ippTools/src/chiptool.c (revision 17591) @@ -105,4 +105,567 @@ psMetadata *where = psMetadataAlloc(); + // convert exp_id into a psS64 + if (exp_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "exp_id", 0, "==", (psS64)atoll(exp_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + return false; + } + } + + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + + // default + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("chiptool_find_rawexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (pretend) { + // then stop before running the query + fprintf(stderr, "%s\n", query); + psFree(query); + return true; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // loop over our list of exp_ids + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + bool status; + psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_id"); + psFree(output); + return false; + } + + // queue the exp + if (!pxchipQueueByExpTag(config, exp_id, workdir, label, reduction, expgroup, dvodb, tess_id, end_stage)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue exp_id: %" PRId64, exp_id); + psFree(output); + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_STR(chip_id, config->args, "-chip_id", false, false); + // convert chip_id into a psS64 + if (chip_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "chip_id", 0, "==", (psS64)atoll(chip_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item chip_id"); + psFree(where); + return false; + } + } + + PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", false, false); + // convert exp_id into a psS64 + if (exp_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "exp_id", 0, "==", (psS64)atoll(exp_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + return false; + } + } + + PXOPT_COPY_STR(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + where = NULL; + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + if ((!state) && (!label)) { + psError(PXTOOLS_ERR_DATA, false, "parameters are required"); + psFree(where); + return false; + } + + if (state) { + // set chipRun.state to state + if (!pxchipRunSetStateByQuery(config, where, state)) { + psFree(where); + return false; + } + } + + if (label) { + // set chipRun.label to label + if (!pxchipRunSetLabelByQuery(config, where, label)) { + psFree(where); + return false; + } + } + + psFree(where); + + return true; +} + + +static bool pendingimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_STR(chip_id, config->args, "-chip_id", false, false); + // convert chip_id into a psS64 + if (chip_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "chipRun.chip_id", 0, "==", (psS64)atoll(chip_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item chip_id"); + psFree(where); + return false; + } + } + + PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", false, false); + // convert exp_id into a psS64 + if (exp_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "rawExp.exp_id", 0, "==", (psS64)atoll(exp_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + return false; + } + } + + PXOPT_COPY_STR(config->args, where, "-class_id", "rawImfile.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + + psString query = pxDataGet("chiptool_pendingimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipPendingImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // chip_id, ext_tag, class_id are required + PXOPT_LOOKUP_STR(chip_id, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + // optional + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_F32(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F32(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F32(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F32(bias, config->args, "-bias", false, false); + PXOPT_LOOKUP_F32(bias_stdev, config->args, "-bias_stdev", false, false); + PXOPT_LOOKUP_F32(fringe_0, config->args, "-fringe_0", false, false); + PXOPT_LOOKUP_F32(fringe_1, config->args, "-fringe_1", false, false); + PXOPT_LOOKUP_F32(fringe_2, config->args, "-fringe_2", false, false); + PXOPT_LOOKUP_F32(sigma_ra, config->args, "-sigma_ra", false, false); + PXOPT_LOOKUP_F32(sigma_dec, config->args, "-sigma_dec", false, false); + PXOPT_LOOKUP_F32(ap_resid, config->args, "-ap_resid", false, false); + PXOPT_LOOKUP_F32(ap_resid_stdev, config->args, "-ap_resid_stdev", false, false); + PXOPT_LOOKUP_F32(zp_mean, config->args, "-zp_mean", false, false); + PXOPT_LOOKUP_F32(zp_stdev, config->args, "-zp_stdev", false, false); + PXOPT_LOOKUP_F32(fwhm_major, config->args, "-fwhm_major", false, false); + PXOPT_LOOKUP_F32(fwhm_minor, config->args, "-fwhm_minor", false, false); + PXOPT_LOOKUP_F32(dtime_detrend, config->args, "-dtime_detrend", false, false); + PXOPT_LOOKUP_F32(dtime_photom, config->args, "-dtime_photom", false, false); + PXOPT_LOOKUP_F32(dtime_astrom, config->args, "-dtime_astrom", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_F32(n_stars, config->args, "-n_stars", false, false); + PXOPT_LOOKUP_F32(n_extended, config->args, "-n_extended", false, false); + PXOPT_LOOKUP_F32(n_cr, config->args, "-n_cr", false, false); + PXOPT_LOOKUP_F32(n_astrom, config->args, "-n_astrom", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!chipProcessedImfileInsert(config->dbh, + (psS64)atoll(chip_id), + (psS64)atoll(exp_id), + class_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + bias, + bias_stdev, + fringe_0, + fringe_1, + fringe_2, + sigma_ra, + sigma_dec, + ap_resid, + ap_resid_stdev, + zp_mean, + zp_stdev, + fwhm_major, + fwhm_minor, + dtime_detrend, + dtime_photom, + dtime_astrom, + hostname, + n_stars, + n_extended, + n_cr, + n_astrom, + path_base, + code + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // XXX I've decided to make the transaction cover the Exp migration as + // well. Otherwise, if the last imfile in an exp is moved and the exp + // migration fails then the data base is left in a satuation where the exp + // migration can't happen. + + if (!chipProcessedCompleteExp(config)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_STR(chip_id, config->args, "-chip_id", false, false); + // convert chip_id into a psS64 + if (chip_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "chipRun.chip_id", 0, "==", (psS64)atoll(chip_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item chip_id"); + psFree(where); + return false; + } + } + + PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", false, false); + // convert exp_id into a psS64 + if (exp_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "rawExp.exp_id", 0, "==", (psS64)atoll(exp_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + return false; + } + } + + PXOPT_COPY_STR(config->args, where, "-class_id", "rawImfile.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + + psString query = pxDataGet("chiptool_processedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "chipProcessedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND chipProcessedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipProcessedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool revertprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_STR(chip_id, config->args, "-chip_id", false, false); + // convert chip_id into a psS64 + if (chip_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "chip_id", 0, "==", (psS64)atoll(chip_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item chip_id"); + psFree(where); + return false; + } + } + + PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", false, false); // convert exp_id into a psS64 if (exp_id) { @@ -158,563 +721,4 @@ } - PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); - PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); - PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); - PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); - PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); - - // default - PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); - PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); - - // find the exp_id of all the exposures that we want to queue up. - psString query = pxDataGet("chiptool_find_rawexp.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - psFree(where); - return false; - } - - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - psFree(where); - - if (pretend) { - // then stop before running the query - fprintf(stderr, "%s\n", query); - psFree(query); - return true; - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("chiptool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - // start a transaction so we don't end up with an exp without any associted - // imfiles - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(output); - return false; - } - - // loop over our list of exp_ids - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - - bool status; - psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_id"); - psFree(output); - return false; - } - - // queue the exp - if (!pxchipQueueByExpTag(config, exp_id, workdir, label, reduction, expgroup, dvodb, tess_id, end_stage)) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, - "failed to trying to queue exp_id: %" PRId64, exp_id); - psFree(output); - return false; - } - } - psFree(output); - - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - return true; -} - - -static bool updaterunMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, NULL); - - psMetadata *where = psMetadataAlloc(); - PXOPT_LOOKUP_STR(chip_id, config->args, "-chip_id", false, false); - // convert chip_id into a psS64 - if (chip_id) { - if (!psMetadataAddS64(where, PS_LIST_TAIL, "chip_id", 0, "==", (psS64)atoll(chip_id))) { - psError(PS_ERR_UNKNOWN, false, "failed to add item chip_id"); - psFree(where); - return false; - } - } - - PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", false, false); - // convert exp_id into a psS64 - if (exp_id) { - if (!psMetadataAddS64(where, PS_LIST_TAIL, "exp_id", 0, "==", (psS64)atoll(exp_id))) { - psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); - psFree(where); - return false; - } - } - - PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); - PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); - PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); - PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); - PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); - PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); - PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); - PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); - PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); - PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); - PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); - PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); - PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); - PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); - PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); - PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); - PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">="); - PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<"); - PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); - PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); - PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); - PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); - PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); - PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); - PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); - PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); - PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); - PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); - PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); - PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); - PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); - - if (!psListLength(where->list) - && !psMetadataLookupBool(NULL, config->args, "-all")) { - psFree(where); - where = NULL; - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - - PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - - if ((!state) && (!label)) { - psError(PXTOOLS_ERR_DATA, false, "parameters are required"); - psFree(where); - return false; - } - - if (state) { - // set chipRun.state to state - if (!pxchipRunSetStateByQuery(config, where, state)) { - psFree(where); - return false; - } - } - - if (label) { - // set chipRun.label to label - if (!pxchipRunSetLabelByQuery(config, where, label)) { - psFree(where); - return false; - } - } - - psFree(where); - - return true; -} - - -static bool pendingimfileMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_LOOKUP_STR(chip_id, config->args, "-chip_id", false, false); - // convert chip_id into a psS64 - if (chip_id) { - if (!psMetadataAddS64(where, PS_LIST_TAIL, "chipRun.chip_id", 0, "==", (psS64)atoll(chip_id))) { - psError(PS_ERR_UNKNOWN, false, "failed to add item chip_id"); - psFree(where); - return false; - } - } - - PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", false, false); - // convert exp_id into a psS64 - if (exp_id) { - if (!psMetadataAddS64(where, PS_LIST_TAIL, "rawExp.exp_id", 0, "==", (psS64)atoll(exp_id))) { - psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); - psFree(where); - return false; - } - } - - PXOPT_COPY_STR(config->args, where, "-class_id", "rawImfile.class_id", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.telescope", "=="); - PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); - - psString query = pxDataGet("chiptool_pendingimfile.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("chiptool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - if (!convertIdToStr(output)) { - psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); - psFree(output); - return false; - } - - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "chipPendingImfile", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - - psFree(output); - - return true; -} - - -static bool addprocessedimfileMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // chip_id, ext_tag, class_id are required - PXOPT_LOOKUP_STR(chip_id, config->args, "-chip_id", true, false); - PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", true, false); - PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); - - // optional - PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); - PXOPT_LOOKUP_F32(bg, config->args, "-bg", false, false); - PXOPT_LOOKUP_F32(bg_stdev, config->args, "-bg_stdev", false, false); - PXOPT_LOOKUP_F32(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); - PXOPT_LOOKUP_F32(bias, config->args, "-bias", false, false); - PXOPT_LOOKUP_F32(bias_stdev, config->args, "-bias_stdev", false, false); - PXOPT_LOOKUP_F32(fringe_0, config->args, "-fringe_0", false, false); - PXOPT_LOOKUP_F32(fringe_1, config->args, "-fringe_1", false, false); - PXOPT_LOOKUP_F32(fringe_2, config->args, "-fringe_2", false, false); - PXOPT_LOOKUP_F32(sigma_ra, config->args, "-sigma_ra", false, false); - PXOPT_LOOKUP_F32(sigma_dec, config->args, "-sigma_dec", false, false); - PXOPT_LOOKUP_F32(ap_resid, config->args, "-ap_resid", false, false); - PXOPT_LOOKUP_F32(ap_resid_stdev, config->args, "-ap_resid_stdev", false, false); - PXOPT_LOOKUP_F32(zp_mean, config->args, "-zp_mean", false, false); - PXOPT_LOOKUP_F32(zp_stdev, config->args, "-zp_stdev", false, false); - PXOPT_LOOKUP_F32(fwhm_major, config->args, "-fwhm_major", false, false); - PXOPT_LOOKUP_F32(fwhm_minor, config->args, "-fwhm_minor", false, false); - PXOPT_LOOKUP_F32(dtime_detrend, config->args, "-dtime_detrend", false, false); - PXOPT_LOOKUP_F32(dtime_photom, config->args, "-dtime_photom", false, false); - PXOPT_LOOKUP_F32(dtime_astrom, config->args, "-dtime_astrom", false, false); - PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); - PXOPT_LOOKUP_F32(n_stars, config->args, "-n_stars", false, false); - PXOPT_LOOKUP_F32(n_extended, config->args, "-n_extended", false, false); - PXOPT_LOOKUP_F32(n_cr, config->args, "-n_cr", false, false); - PXOPT_LOOKUP_F32(n_astrom, config->args, "-n_astrom", false, false); - PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); - - // default values - PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); - - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - if (!chipProcessedImfileInsert(config->dbh, - (psS64)atoll(chip_id), - (psS64)atoll(exp_id), - class_id, - uri, - bg, - bg_stdev, - bg_mean_stdev, - bias, - bias_stdev, - fringe_0, - fringe_1, - fringe_2, - sigma_ra, - sigma_dec, - ap_resid, - ap_resid_stdev, - zp_mean, - zp_stdev, - fwhm_major, - fwhm_minor, - dtime_detrend, - dtime_photom, - dtime_astrom, - hostname, - n_stars, - n_extended, - n_cr, - n_astrom, - path_base, - code - )) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - // XXX I've decided to make the transaction cover the Exp migration as - // well. Otherwise, if the last imfile in an exp is moved and the exp - // migration fails then the data base is left in a satuation where the exp - // migration can't happen. - - if (!chipProcessedCompleteExp(config)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - return true; -} - - -static bool processedimfileMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_LOOKUP_STR(chip_id, config->args, "-chip_id", false, false); - // convert chip_id into a psS64 - if (chip_id) { - if (!psMetadataAddS64(where, PS_LIST_TAIL, "chipRun.chip_id", 0, "==", (psS64)atoll(chip_id))) { - psError(PS_ERR_UNKNOWN, false, "failed to add item chip_id"); - psFree(where); - return false; - } - } - - PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", false, false); - // convert exp_id into a psS64 - if (exp_id) { - if (!psMetadataAddS64(where, PS_LIST_TAIL, "rawExp.exp_id", 0, "==", (psS64)atoll(exp_id))) { - psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); - psFree(where); - return false; - } - } - - PXOPT_COPY_STR(config->args, where, "-class_id", "rawImfile.class_id", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.telescope", "=="); - PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); - - psString query = pxDataGet("chiptool_processedimfile.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "chipProcessedImfile"); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - if (faulted) { - // list only faulted rows - psStringAppend(&query, " %s", "AND chipProcessedImfile.fault != 0"); - } else { - // don't list faulted rows - psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0"); - } - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("chiptool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - if (!convertIdToStr(output)) { - psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); - psFree(output); - return false; - } - - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "chipProcessedImfile", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - - psFree(output); - - return true; -} - -static bool revertprocessedimfileMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_LOOKUP_STR(chip_id, config->args, "-chip_id", false, false); - // convert chip_id into a psS64 - if (chip_id) { - if (!psMetadataAddS64(where, PS_LIST_TAIL, "chip_id", 0, "==", (psS64)atoll(chip_id))) { - psError(PS_ERR_UNKNOWN, false, "failed to add item chip_id"); - psFree(where); - return false; - } - } - - PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", false, false); - // convert exp_id into a psS64 - if (exp_id) { - if (!psMetadataAddS64(where, PS_LIST_TAIL, "exp_id", 0, "==", (psS64)atoll(exp_id))) { - psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); - psFree(where); - return false; - } - } - - PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); - PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); - PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); - PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); - PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); - PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); - PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); - PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); - PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); - PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); - PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); - PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); - PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); - PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); - PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); - PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); - PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">="); - PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<"); - PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); - PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); - PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); - PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); - PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); - PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); - PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); - PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); - PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); - PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); - PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); - PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); - PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); - - if (!psListLength(where->list) - && !psMetadataLookupBool(NULL, config->args, "-all")) { - psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - psString query = pxDataGet("chiptool_revertprocessedimfile.sql"); if (!query) {