Index: trunk/ippTools/src/dettool.c =================================================================== --- trunk/ippTools/src/dettool.c (revision 8383) +++ trunk/ippTools/src/dettool.c (revision 8406) @@ -43,4 +43,5 @@ static detNormalizedStatImfileRow *detStackedToDetNormalizedStatImfile(pxConfig *config, detStackedImfileRow *stackedImfile); static detNormalizedImfileRow *detNormalizedStatToDetNormalizedmfile(pxConfig *config, detNormalizedStatImfileRow *statImfile); +static detResidImfileRow *detNormalizedToDetResidImfile(pxConfig *config, detNormalizedImfileRow *normalizedImfile); static psArray *validDetInputClassIds(pxConfig *config, const char *det_id); static psArray *searchInputImfiles(pxConfig *config, const char *det_id); @@ -2357,8 +2358,4 @@ } -static bool toresidexpMode(pxConfig *config) -{ - return true; -} static bool normalizedframeMode(pxConfig *config) @@ -2449,31 +2446,172 @@ PS_ASSERT_PTR_NON_NULL(config, false); - // det_id, exp_id, class_id, & recipe are required + // make sure that there is a coresponding entry in detNormalizedImfile + // and the exp_id specified is valid + // select * from detNormalizedImfile + // by det_id, iteration, class_id + // where det_id, iteration, class_id is not in detResidImfile + psString query = psStringCopy( + "SELECT" + " detNormalizedImfile.*" + " FROM detNormalizedImfile" + " LEFT JOIN detInputExp" + " USING(det_id, iteration)" + " LEFT JOIN detResidImfile" + " ON detNormalizedImfile.det_id = detResidImfile.det_id" + " AND detNormalizedImfile.iteration= detResidImfile.iteration" + " AND detInputExp.exp_id = detResidImfile.exp_id" + " AND detNormalizedImfile.class_id = detResidImfile.class_id" + " WHERE" + " detInputExp.exp_id = '%s'" + " AND detResidImfile.det_id IS NULL" + " AND detResidImfile.iteration IS NULL" + " AND detResidImfile.exp_id IS NULL" + " AND detResidImfile.class_id IS NULL" + ); + + { + // build a query to search by det_id, iteration, class_id + psMetadata *where = psMetadataAlloc(); + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + psFree(where); + psFree(query); + return false; + } + if (det_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "det_id", 0, "==", det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + psFree(query); + return false; + } + } + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + psFree(where); + psFree(query); + return false; + } + // always set iteration + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", iteration)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + psFree(query); + return false; + } + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + psFree(where); + psFree(query); + return false; + } + if (class_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "class_id", 0, "==", class_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class_id"); + psFree(where); + psFree(query); + return false; + } + } + + // there's not + psString whereClaus = psDBGenerateWhereConditionSQL(where, "detNormalizedImfile"); + psFree(where); + if (whereClaus) { + psStringAppend(&query, " AND %s", whereClaus); + psFree(whereClaus); + } + } + + // exp_id is manadatory to cross check that this is a exp_id for this + // detRUn bool status = false; - psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); - return false; - } - if (!det_id) { - psError(PS_ERR_UNKNOWN, true, "-det_id is required"); - return false; - } psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id"); if (!status) { psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id"); + psFree(query); return false; } if (!exp_id) { psError(PS_ERR_UNKNOWN, true, "-exp_id is required"); - return false; - } - psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); - return false; - } - if (!class_id) { - psError(PS_ERR_UNKNOWN, true, "-class_id is required"); + psFree(query); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query, exp_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no pending detNormalizedImfile rows found"); + return false; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + // convert metadata into a detNormalizedImfile object + detNormalizedImfileRow *normalizedImfile = detNormalizedImfileObjectFromMetadata(row); + // convert detNormalizedImfile object into a detResidImfile + detResidImfileRow *residImfile = detNormalizedToDetResidImfile(config, normalizedImfile); + psFree(normalizedImfile); + if (!residImfile) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to convert detNormalizedImfile to detResidImfile"); + psFree(output); + return false; + } + // insert detResidImfile object into the database + if (!detResidImfileInsertObject(config->dbh, residImfile)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(residImfile); + psFree(output); + } + psFree(residImfile); + } + + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static detResidImfileRow *detNormalizedToDetResidImfile(pxConfig *config, detNormalizedImfileRow *normalizedImfile) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(normalizedImfile, NULL); + + bool status = false; + psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id"); + return false; + } + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "-exp_id is required"); return false; } @@ -2508,79 +2646,15 @@ } - // iteration has a default value - psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); - return false; - } - - // check det_id, exp_id, & class_id against the input rawImfiles - - // XXX what we really want here is to do a join but psDB doesn't support - // that type of operation - // we have to generate our own where claus as we want to exclude -stats - // from the search - psMetadata *where = psMetadataAlloc(); - if (!psMetadataAddS32(where, PS_LIST_TAIL, "det_id", 0, "==", - (psS32)atoi(det_id))) { - psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); - psFree(where); - return false; - } - if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", 0, "==", exp_id)) { - psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); - psFree(where); - return false; - } - - psArray *rawImfiles = searchRawImfiles(config, where); - psFree(where); - if (!rawImfiles) { - psError(PS_ERR_UNKNOWN, true, - "det_id %s, exp_id %s, class_id %s does not corespond to an input imfile", - det_id, exp_id, class_id); - return false; - } - - // iterate through rawImfiles looking for a match to our class_id - bool found = false; - for (long i = 0; i < psArrayLength(rawImfiles); i++) { - if (strcmp(class_id, - ((rawImfileRow *)rawImfiles->data[i])->class_id) == 0) { - found = true; - break; - } - } - - psFree(rawImfiles); - - if (!found) { - psError(PS_ERR_UNKNOWN, true, - "det_id %s, exp_id %s, class_id %s does not corespond to an input imfile", - det_id, exp_id, class_id); - return false; - } - // create a new detResidImfileRow and insert it - detResidImfileRow *residImfile = detResidImfileRowAlloc( - (psS32)atol(det_id), - iteration, - exp_id, - class_id, - recipe, - uri, - b1_uri, - b2_uri - ); - - if (!detResidImfileInsertObject(config->dbh, residImfile)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(residImfile); - return false; - } - - psFree(residImfile); - - return true; + return detResidImfileRowAlloc( + normalizedImfile->det_id, + normalizedImfile->iteration, + exp_id, + normalizedImfile->class_id, + recipe, + uri, + b1_uri, + b2_uri + ); } @@ -2617,4 +2691,103 @@ psFree(residImfiles); + + return true; +} + +static bool toresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + /* +which returns a list of exposures for which all component class ids +have had residuals created. This list includes the detrend id, +iteration, exposure id, detrend type, and whether the exposure was +included in the stack for this iteration. + */ + + + // select detRun.position + // select detRun.iteration + // select detRun.det_type + // select detInputExp.exp_id + // select detInputExp.include + // by: + // find the current iteration bassed on det_id + // find all exp_ids in the current det_id/iteration from detInputExp + // compare to detInputExp.imfiles to derResidImfile by class_id + // and: + // detResidImfile.{det_id, iteration, exp_id} is not in detResidExp + + psString query = psStringCopy( + "SELECT DISTINCT" + " detRun.position AS det_id," + " detRun.iteration," + " detRun.det_type," + " detInputExp.exp_id," + " detInputExp.include" + " FROM detRun" + " LEFT JOIN detInputExp" + " ON detRun.position = detInputExp.det_id" + " AND detRun.iteration = detInputExp.iteration" + " LEFT JOIN rawDetrendExp" + " ON detInputExp.exp_id = rawDetrendExp.exp_id" + " LEFT JOIN detResidImfile" + " ON detRun.position = detResidImfile.det_id" + " AND detRun.iteration = detResidImfile.iteration" + " AND detInputExp.exp_id = detResidImfile.exp_id" + " LEFT JOIN detResidExp" + " ON detRun.position = detResidExp.det_id" + " AND detRun.iteration = detResidExp.iteration" + " AND detInputExp.exp_id = detResidExp.exp_id" + " GROUP BY" + " detInputExp.exp_id" + " WHERE" + " detResidExp.det_id IS NULL" + " AND detResidExp.iteration IS NULL" + " AND detResidExp.exp_id IS NULL" + " HAVING rawDetrendExp.imfiles = COUNT(detStackedImfile.class_id)" + ); + + // XXX does it make sens to accept any search params? +#if 0 + if (config->where) { + psString whereClaus = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClaus); + psFree(whereClaus); + } +#endif + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no pending rawDetrendExp rows found"); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawDetrendImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); return true;