Index: /trunk/ippTools/src/dettool.c =================================================================== --- /trunk/ippTools/src/dettool.c (revision 8450) +++ /trunk/ippTools/src/dettool.c (revision 8451) @@ -44,4 +44,5 @@ static detNormalizedImfileRow *detNormalizedStatToDetNormalizedmfile(pxConfig *config, detNormalizedStatImfileRow *statImfile); static detResidImfileRow *detNormalizedToDetResidImfile(pxConfig *config, detNormalizedImfileRow *normalizedImfile); +static detResidExpRow *mdToDetResidExp(pxConfig *config, psMetadata *row); static psArray *validDetInputClassIds(pxConfig *config, const char *det_id); static psArray *searchInputImfiles(pxConfig *config, const char *det_id); @@ -706,6 +707,5 @@ iteration, rawExp->exp_id, - true, // use - false // accept + true // use ); } @@ -2834,6 +2834,260 @@ { PS_ASSERT_PTR_NON_NULL(config, false); + + /* +which returns a list of exposures for which all component class ids +have had residuals created. This list includes the detrend id, +iteration, exposure id, detrend type, and whether the exposure was +included in the stack for this iteration. + */ + + + // select detRun.position + // select detRun.iteration + // select detRun.det_type + // select detInputExp.exp_id + // select detInputExp.include + // by: + // find the current iteration bassed on det_id + // find all exp_ids in the current det_id/iteration from detInputExp + // compare to detInputExp.imfiles to derResidImfile by class_id + // and: + // detResidImfile.{det_id, iteration, exp_id} is not in detResidExp + + psString query = psStringCopy( + "SELECT DISTINCT" + " det_id," + " iteration," + " exp_id," + " include" + " FROM" + " (SELECT DISTINCT" + " detRun.position AS det_id," + " detRun.iteration," + " detRun.det_type," + " detInputExp.exp_id," + " detInputExp.include," + " rawDetrendExp.imfiles" + " FROM detRun" + " LEFT JOIN detInputExp" + " ON detRun.position = detInputExp.det_id" + " AND detRun.iteration = detInputExp.iteration" + " LEFT JOIN rawDetrendExp" + " ON detInputExp.exp_id = rawDetrendExp.exp_id" + " LEFT JOIN detResidImfile" + " ON detRun.position = detResidImfile.det_id" + " AND detRun.iteration = detResidImfile.iteration" + " AND detInputExp.exp_id = detResidImfile.exp_id" + " LEFT JOIN detResidExp" + " ON detRun.position = detResidExp.det_id" + " AND detRun.iteration = detResidExp.iteration" + " AND detInputExp.exp_id = detResidExp.exp_id" + " WHERE" + " detResidExp.det_id IS NULL" + " AND detResidExp.iteration IS NULL" + " AND detResidExp.exp_id IS NULL" + " GROUP BY" + " detInputExp.exp_id" + " HAVING" + " rawDetrendExp.imfiles = COUNT(detResidImfile.class_id)" + " ) AS toresidexp" + ); + + { + // build a query to search by det_id, iteration, exp_id + psMetadata *where = psMetadataAlloc(); + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + psFree(where); + psFree(query); + return false; + } + if (det_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "det_id", 0, "==", det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + psFree(query); + return false; + } + } + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + psFree(where); + psFree(query); + return false; + } + // always set iteration + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", iteration)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + psFree(query); + return false; + } + psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id"); + psFree(where); + psFree(query); + return false; + } + if (exp_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", 0, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + psFree(query); + return false; + } + } + + // there's not + psString whereClaus = psDBGenerateWhereConditionSQL(where, "detNormalizedImfile"); + psFree(where); + if (whereClaus) { + psStringAppend(&query, " AND %s", whereClaus); + psFree(whereClaus); + } + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no pending rawDetrendExp rows found"); + return false; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + // convert metadata into a detResidExp object + detResidExpRow *residExp = mdToDetResidExp(config, row); + if (!residExp) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata to detResidExp"); + psFree(output); + return false; + } + // insert detResidExp object into the database + if (!detResidExpInsertObject(config->dbh, residExp)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(residExp); + psFree(output); + } + psFree(residExp); + } + + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static detResidExpRow *mdToDetResidExp(pxConfig *config, psMetadata *row) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(row, NULL); + + bool status = false; + // values from row + psS32 det_id = psMetadataLookupS32(&status, row, "det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for det_id"); + return false; + } + if (isnan(det_id)) { + psError(PS_ERR_UNKNOWN, true, "det_id is required"); + return false; + } + psS32 iteration = psMetadataLookupS32(&status, row, "iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for iteration"); + return false; + } + psString exp_id = psMetadataLookupStr(&status, row, "exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_id"); + return false; + } + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "exp_id is required"); + return false; + } + + // values from config + psF32 stats = psMetadataLookupF32(&status, config->args, "-stats"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -stats"); + return false; + } + if (isnan(stats)) { + psError(PS_ERR_UNKNOWN, true, "-stats is required"); + return false; + } + // optional + bool reject = psMetadataLookupBool(&status, config->args, "-reject"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -reject"); + return false; + } + psString b1_uri = psMetadataLookupStr(&status, config->args, "-b1_uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -b1_uri"); + return false; + } + if (!b1_uri) { + psError(PS_ERR_UNKNOWN, true, "-b1_uri is required"); + return false; + } + psString b2_uri = psMetadataLookupStr(&status, config->args, "-b2_uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -b2_uri"); + return false; + } + if (!b2_uri) { + psError(PS_ERR_UNKNOWN, true, "-b2_uri is required"); + return false; + } + + // create a new detResidImfileRow and insert it + return detResidExpRowAlloc( + det_id, + iteration, + exp_id, + stats, + !reject, + b1_uri, + b2_uri + ); +} + +#if 0 + PS_ASSERT_PTR_NON_NULL(config, false); // det_id, exp_id, & recipe are required + bool status = false; psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); @@ -2855,11 +3109,11 @@ return false; } - psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); - return false; - } - if (!class_id) { - psError(PS_ERR_UNKNOWN, true, "-class_id is required"); + psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id"); + return false; + } + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "-exp_id is required"); return false; } @@ -2935,4 +3189,5 @@ return true; } +#endif static bool residexpMode(pxConfig *config) @@ -3226,6 +3481,5 @@ newIteration, inputExp->exp_id, - true, // use - true // accept + true // use ); psArrayAdd(newInputExps, 0, newInputExp); Index: /trunk/ippTools/src/dettoolConfig.c =================================================================== --- /trunk/ippTools/src/dettoolConfig.c (revision 8450) +++ /trunk/ippTools/src/dettoolConfig.c (revision 8451) @@ -167,8 +167,14 @@ psMetadataAddStr(addstacArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID (required)", NULL); + psMetadataAddStr(addstacArgs, PS_LIST_TAIL, "-recip", 0, + "define recipe (required)", NULL); + psMetadataAddF64(addstacArgs, PS_LIST_TAIL, "-bg", 0, + "define exposue background", NAN); + psMetadataAddF64(addstacArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposue background stdev", NAN); + psMetadataAddF64(addstacArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposue background mean stdev", NAN); psMetadataAddStr(addstacArgs, PS_LIST_TAIL, "-uri", 0, "define URI (required)", NULL); - psMetadataAddStr(addstacArgs, PS_LIST_TAIL, "-recip", 0, - "define recipe (required)", NULL); psMetadataAddBool(addstacArgs, PS_LIST_TAIL, "-pleasenormalize", 0, "dude, make me normal", false); @@ -278,8 +284,14 @@ psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID (required)", NULL); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-uri", 0, + "define resid file URI (required)", NULL); psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-recip", 0, "define recipe (required)", NULL); - psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-uri", 0, - "define resid file URI (required)", NULL); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg", 0, + "define exposue background", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposue background stdev", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposue background mean stdev", NAN); psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-b1_uri", 0, "define banana 1", NULL); @@ -310,8 +322,6 @@ psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-exp_id", 0, "define detrend ID (required)", NULL); - psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-b1_uri", 0, - "define banana 1", NULL); - psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-b2_uri", 0, - "define banana 2", NULL); + psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-recip", 0, + "define recipe (required)", NULL); psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg", 0, "define exposue background", NAN); @@ -320,4 +330,8 @@ psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposue background mean stdev", NAN); + psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-b1_uri", 0, + "define banana 1", NULL); + psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-b2_uri", 0, + "define banana 2", NULL); psMetadataAddBool(addresidexpArgs, PS_LIST_TAIL, "-reject", 0, "exposure is not to be stacked in the next iteration", false);