Index: trunk/ippTools/src/magictool.c =================================================================== --- trunk/ippTools/src/magictool.c (revision 18658) +++ trunk/ippTools/src/magictool.c (revision 18663) @@ -110,12 +110,16 @@ PS_ASSERT_PTR_NON_NULL(config, false); + // Required PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); + + // Optional PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); - - // create warped skycells temp table + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // Create temporary table of the best diffs { - psString query = pxDataGet("magictool_create_tmp_warpcomplete.sql"); + psString query = pxDataGet("magictool_definebyquery_temp_create.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -127,13 +131,36 @@ return false; } - } - - // find warped skycells + psFree(query); + } + + // Insert list of best diffs into temporary table { - psString query = pxDataGet("magictool_find_complete_warpruns.sql"); + psString query = pxDataGet("magictool_definebyquery_temp_insert.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); return false; } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); + PXOPT_COPY_S64(config->args, where, "-good_frac", "diffSkyfile.good_frac", ">="); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + psString groupby = pxDataGet("magictool_definebyquery_temp_insert_groupby.sql"); + if (!groupby) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(query); + return false; + } + + psStringAppend(&query, " %s", groupby); + psFree(groupby); if (!p_psDBRunQuery(config->dbh, query)) { @@ -145,11 +172,33 @@ } - // find the diff_id's of the warped skycells + // Get list of exposures ready to magic { - psString query = pxDataGet("magictool_find_complete_diffed_exposures.sql"); + psString query = pxDataGet("magictool_definebyquery_select_part1.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); return false; } + + { + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + } + + psString part2 = pxDataGet("magictool_definebyquery_select_part2.sql"); + if (!part2) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psStringAppend(&query, " %s", part2); + psFree(part2); if (!p_psDBRunQuery(config->dbh, query)) { @@ -180,91 +229,71 @@ } - // pre-allocate enough hash buckets for 1 unique warp_id per row - psHash *groups = psHashAlloc(psArrayLength(output)); - // iterate over result set, and group by warp_id + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psString insert = pxDataGet("magictool_definebyquery_insert.sql"); // Insert query + + // iterate over array of grouped warp_ids for (long i = 0; i < psArrayLength(output); i++) { - // lookup the key we're grouping by - bool status = false; - psS64 warp_id = psMetadataLookupS64(&status, output->data[i], "warp_id"); - if (!status) { - psAbort("failed to lookup value for warp_id"); - } - - psString key = psDBIntToString(warp_id); - // look to see if there is a bin for this key - psArray *groupBin = psHashLookup(groups, key); - if (!groupBin) { - // if not, create one - groupBin = psArrayAlloc(0); - psHashAdd(groups, key, groupBin); - } - psFree(key); - - // add this row to the bin - psArrayAdd(groupBin, 0, output->data[i]); - psFree(groupBin); - } - psFree(output); - - // create a magic run - psArray *grouped = psHashToArray(groups); - psFree(groups); - - // iterate over array of grouped warp_ids - for (long i = 0; i < psArrayLength(grouped); i++) { - psArray *group = grouped->data[i]; + psMetadata *row = output->data[i]; // Row of interest + psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier // create a new magicRun for this group - magicRunRow *run = magicRunRowAlloc( - 0, // ID - "reg", // state - workdir, // workdir - "dirty", // workdir_state - label, // label - dvodb, // dvodb - registered, // registered - 0 // fault - ); + magicRunRow *run = magicRunRowAlloc(0, exp_id, "reg", workdir, "dirty", label, dvodb, registered, 0); if (!run) { psAbort("failed to alloc magicRun object"); } + if (!magicRunInsertObject(config->dbh, run)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(run); - psFree(grouped); - return false; - } - - // get the assigned warp_id - psS64 magic_id = psDBLastInsertID(config->dbh); - - // insert all rows in this bin into the new magicRun - for (long j = 0; j < psArrayLength(group); j++) { - psMetadata *row = group->data[j]; - - bool status = false; - psS64 diff_id = psMetadataLookupS64(&status, row, "diff_id"); - if (!status) { - psAbort("failed to lookup value for diff_id"); + psFree(insert); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); } - psString node = psMetadataLookupStr(&status, row, "skycell_id"); - if (!status) { - psAbort("failed to lookup value for diff_id"); + return false; + } + if (!magicRunPrintObject(stdout, run, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(run); + psFree(insert); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); } - - if (!magicInputSkyfileInsert(config->dbh, - magic_id, - diff_id, - node)) { + return false; + } + psFree(run); + + psS64 magic_id = psDBLastInsertID(config->dbh); // Assigned identifier + + // Create a suitable insertion query for this run + psString thisInsert = psStringCopy(insert); + psString idString = NULL; + psStringAppend(&idString, "%" PRId64, magic_id); + psStringSubstitute(&thisInsert, idString, "@MAGIC_ID@"); + psFree(idString); + + if (!p_psDBRunQuery(config->dbh, thisInsert, magic_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(thisInsert); + psFree(insert); + psFree(output); + if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(grouped); - return false; } - } - - // set magicRun.state to 'run' - return setmagicRunState(config, magic_id, "run"); - } - psFree(grouped); + return false; + } + psFree(thisInsert); + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(output); return true; @@ -277,4 +306,5 @@ // required PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); // optional @@ -286,4 +316,5 @@ magicRunRow *run = magicRunRowAlloc( 0, // ID + exp_id, "reg", // state workdir, Index: trunk/ippTools/src/magictoolConfig.c =================================================================== --- trunk/ippTools/src/magictoolConfig.c (revision 18658) +++ trunk/ippTools/src/magictoolConfig.c (revision 18663) @@ -48,13 +48,17 @@ // -definebyquery - psMetadata *queueArgs = psMetadataAlloc(); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); - psMetadataAddTime(queueArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "limit good_frac", NAN); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -definerun psMetadata *definerunArgs = psMetadataAlloc(); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-exp_id", 0, "define exp_id (required)", 0); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); @@ -152,5 +156,5 @@ psMetadata *modes = psMetadataAlloc(); - PXOPT_ADD_MODE("-definebyquery", "", MAGICTOOL_MODE_DEFINEBYQUERY, queueArgs); + PXOPT_ADD_MODE("-definebyquery", "", MAGICTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); PXOPT_ADD_MODE("-definerun", "", MAGICTOOL_MODE_DEFINERUN, definerunArgs); PXOPT_ADD_MODE("-updaterun", "", MAGICTOOL_MODE_UPDATERUN, updaterunArgs);