Index: trunk/ippTools/src/regtool.c =================================================================== --- trunk/ippTools/src/regtool.c (revision 8188) +++ trunk/ippTools/src/regtool.c (revision 8250) @@ -8,6 +8,8 @@ #include "p0search.h" -static bool pendingMode(pxConfig *config); -static bool updateMode(pxConfig *config); +static bool pendingexpMode(pxConfig *config); +static bool pendingimfileMode(pxConfig *config); +static bool updateexpMode(pxConfig *config); +static bool updateimfileMode(pxConfig *config); static psArray *newFrameSearchPending(pxConfig *config); static p1PendingExpRow *newToP1PendingExp(newExpRow *newExp); @@ -16,4 +18,6 @@ static rawScienceFrame *newToRawScienceFrame(pxConfig *config, newFrame *newFrame); static rawScienceExpRow *newToRawScienceExp(pxConfig *config, newExpRow *exp); +static rawDetrendExpRow *newToRawDetrendExp(pxConfig *config, newExpRow *exp); +static rawImfileRow *newToRawImfile(pxConfig *config, newImfileRow *exp); @@ -32,6 +36,8 @@ switch (config->mode) { - MODECASE(P0SEARCH_MODE_PENDING, pendingMode); - MODECASE(P0SEARCH_MODE_UPDATE, updateMode); + MODECASE(P0SEARCH_MODE_PENDINGEXP, pendingexpMode); + MODECASE(P0SEARCH_MODE_PENDINGIMFILE, pendingimfileMode); + MODECASE(P0SEARCH_MODE_UPDATEEXP, updateexpMode); + MODECASE(P0SEARCH_MODE_UPDATEIMFILE, updateimfileMode); default: psAbort(argv[0], "invalid option (this should not happen)"); @@ -52,29 +58,135 @@ } -static bool pendingMode(pxConfig *config) +static bool pendingexpMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); - psArray *new = newFrameSearchPending(config); - if (!new) { - psError(PS_ERR_UNKNOWN, false, "no newFrames found"); - return false; - } - - bool status = newFramePrint(stdout, config, new); - if (!status) { - psError(PS_ERR_UNKNOWN, false,"newFramePrint() failed"); - psFree(new); - return false; - } - - psFree(new); + // return only exps that are not in rawScienceExp or rawDetrendExp and have + // ALL of their imfiles in rawImfile (by count) and have no associated + // imfiles left in newImfile + + char *query = + "SELECT newExp.* FROM newExp" + " LEFT JOIN rawScienceExp USING(exp_id)" + " LEFT JOIN rawDetrendExp USING (exp_id)" + " LEFT JOIN newImfile USING (exp_id)" + " WHERE newExp.exp_id is NOT NULL" + " AND rawScienceExp.exp_id IS NULL" + " AND rawDetrendExp.exp_id IS NULL" + " AND newImfile.exp_id IS NULL" + " AND newExp.imfiles = (SELECT COUNT(exp_id)" + " FROM rawImfile GROUP BY exp_id)"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "failed to fetch the database result"); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "newImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); return true; } -static bool updateMode(pxConfig *config) +static bool pendingimfileMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); + + // select newImfiles who's exp_id is in newExp AND don't have their exp_id + // in rawScienceExp or rawDetrendExp + // XXX having the same exp_id in newExp and raw*Exp is probably an error + // that should be checked for + char *query = + "SELECT newImfile.* FROM newImfile" + " LEFT JOIN newExp USING(exp_id)" + " LEFT JOIN rawScienceExp USING(exp_id)" + " LEFT JOIN rawDetrendExp USING (exp_id)" + " WHERE newExp.exp_id is NOT NULL" + " AND rawScienceExp.exp_id IS NULL" + " AND rawDetrendExp.exp_id IS NULL"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "failed to fetch the database result"); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "newImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool updateexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // make sure that the exp_id(s) are ready to be updated based on the same + // critera used in pendingexpMode + char *query = + "SELECT newImfile.* FROM newImfile" + " LEFT JOIN newExp USING(exp_id)" + " LEFT JOIN rawScienceExp USING(exp_id)" + " LEFT JOIN rawDetrendExp USING (exp_id)" + " WHERE newExp.exp_id is NOT NULL" + " AND rawScienceExp.exp_id IS NULL" + " AND rawDetrendExp.exp_id IS NULL"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "failed to fetch the database result"); + return false; + } bool status = false; @@ -85,187 +197,180 @@ } - psArray *new = newFrameSearchPending(config); - if (!new) { - psError(PS_ERR_UNKNOWN, false, "no newFrames found"); - return true; - } - - // insert 'new' rawScience & detrendframes - if (psArrayLength(new) > 0) { - // start a transaction so we don't end up file raw* frames but no - // p2Pending*, etc. + for (long i = 0; psArrayLength(output) > i; i++) { + } + + return true; +} + +static bool updateimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // XXX search by the whole frame some imfiles without a newExp don't get + // processed -- this may not be the correct thing to do + char *query = + "SELECT * FROM" + " (SELECT newImfile.* FROM newImfile" + " LEFT JOIN newExp USING(exp_id)" + " LEFT JOIN rawScienceExp USING(exp_id)" + " LEFT JOIN rawDetrendExp USING (exp_id)" + " WHERE newExp.exp_id IS NOT NULL" + " AND rawScienceExp.exp_id IS NULL" + " AND rawDetrendExp.exp_id IS NULL) AS foo" + " %s"; // WHERE class is generated from exp_id, class, & class_id + + char *whereClaus = NULL; + { + psMetadata *where = psMetadataAlloc(); + bool status = false; + psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id"); + return false; + } + if (exp_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", 0, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + return false; + } + } + psString class = psMetadataLookupStr(&status, config->args, "-class"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class"); + return false; + } + if (class) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "class", 0, "==", class)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class"); + psFree(where); + return false; + } + } + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + return false; + } + if (class_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "class_id", 0, "==", class_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class_id"); + psFree(where); + return false; + } + } + + // there's not + whereClaus = psDBGenerateWhereSQL(where); + psFree(where); + } + + if (!p_psDBRunQuery(config->dbh, query, whereClaus)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereClaus); + return false; + } + psFree(whereClaus); + + psArray *newImfiles = p_psDBFetchResult(config->dbh); + if (!newImfiles) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "failed to fetch the database result"); + return false; + } + + // insert 'newImfile's into rawImfile + if (psArrayLength(newImfiles) > 0) { + // start a transaction so we don't end up half of the imfiles we were + // trying to update uninserted if (!psDBTransaction(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(exp); - psFree(new); + psFree(newImfiles); return false; } - for (long i = 0; i < psArrayLength(new); i++) { - newFrame *newFrame = new->data[i]; - if (detrend) { - // it's a detrend frame - rawDetrendFrame *frame = newToRawDetrendFrame(config, newFrame); - if (!frame) { - psError(PS_ERR_UNKNOWN, false, "failed to convert new frame into a detrend frame"); - psFree(new); - return false; - } - if (!rawDetrendFrameInsert(config, frame)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "failed to insert frame into the database"); - psFree(frame); - psFree(new); - return false; - } - psFree(frame); - continue; - } - - // else - // it's a science frame - // insert into into rawScienceExp - rawScienceFrame *frame = newToRawScienceFrame(config, newFrame); - if (!frame) { + for (long i = 0; i < psArrayLength(newImfiles); i++) { + newImfileRow *object = newImfileObjectFromMetadata(newImfiles->data[i]); + rawImfileRow *imfile = newToRawImfile(config, object); + psFree(object); + if (!imfile) { // rollback if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); } - psError(PS_ERR_UNKNOWN, false, "failed to convert new frame into a science frame"); - psFree(new); + psError(PS_ERR_UNKNOWN, false, "failed to create a new rawImfile row"); + psFree(newImfiles); return false; } - if (!rawScienceFrameInsert(config, frame)) { + if (!rawImfileInsertObject(config->dbh, imfile)) { // rollback if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); } - psError(PS_ERR_UNKNOWN, false, "failed to insert frame into the database"); - psFree(frame); - psFree(new); + psError(PS_ERR_UNKNOWN, false, "failed to insert row into the database"); + psFree(imfile); + psFree(newImfiles); return false; } - psFree(frame); - - // insert into into p2PendingExp - // XXX add a newToP2PendingFrame() - // XXX add a p2PendingFrameInsert() - p2PendingExpRow *exp = newToP2PendingExp(config, newFrame->exposure); - if (!exp) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "failed to convert newExp into a p2PendingExp"); - psFree(new); - return false; - } - - if (!p2PendingExpInsertObject(config->dbh, exp)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "failed to insert p2PendingExp into the database"); - psFree(exp); - psFree(new); - return false; - } - psFree(exp); - - // insert into into p2PendingImfiles - for (long i = 0; i < psArrayLength(newFrame->images); i++) { - p2PendingImfileRow *imfile = newImfileToP2PendingImfile(newFrame->images->data[i]); - if (!p2PendingImfileInsertObject(config->dbh, imfile)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "failed to insert p2PendingImfile into the database"); - psFree(imfile); - psFree(new); - return false; - } - psFree(imfile); - } - } - // point of no return for p2Pending[Exp|Imfile] + psFree(imfile); + } + + // point of no return for rawImfile if (!psDBCommit(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newImfiles); return false; } } - psFree(new); + psFree(newImfiles); return true; } +#if 0 static psArray *newFrameSearchPending(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, NULL); - psArray *new = newFrameSearch(config); - if (!new) { - psError(PS_ERR_UNKNOWN, false, "no newFrames found"); - return false; - } - - psArray *raw = rawScienceFrameSearch(config); - if (!raw) { - psError(PS_ERR_UNKNOWN, false, "no rawScienceFrames found"); - // return false; - } - - psArray *rawDetrend = rawDetrendFrameSearch(config); - if (!rawDetrend) { - psError(PS_ERR_UNKNOWN, false, "no rawDetrendFrames found"); - // return false; - } - - // ignore duplicate raw frames - if (raw) { - for (long i = 0; i < new->n; i++) { - newFrame *newFrame = new->data[i]; - for (long j = 0; j < raw->n; j++) { - rawScienceFrame *rawScienceFrame = raw->data[j]; - if (strcmp(newFrame->exposure->exp_id, - rawScienceFrame->exposure->exp_id) == 0) { - psArrayRemove(new, newFrame); - // dec the counter as the array just got shorter - //and we don't want to skip elemnts - i--; - break; - } - } - } - psFree(raw); - } - - // ignore duplicate rawDetrend frames - if (rawDetrend) { - for (long i = 0; i < new->n; i++) { - newFrame *newFrame = new->data[i]; - for (long j = 0; j < rawDetrend->n; j++) { - rawDetrendFrame *rawDetrendFrame = rawDetrend->data[j]; - if (strcmp(newFrame->exposure->exp_id, - rawDetrendFrame->exposure->exp_id) == 0) { - psArrayRemove(new, newFrame); - // dec the counter as the array just got shorter - //and we don't want to skip elemnts - i--; - break; - } - } - } - psFree(rawDetrend); - } + char *query = "select newImfile.* from newImfile + LEFT JOIN newExp USING(exp_id) + LEFT JOIN rawScienceExp USING(exp_id) + LEFT JOIN rawDetrendExp USING (exp_id) + WHERE newExp.exp_id is NOT NULL + AND rawScienceExp.exp_id IS NULL + AND rawDetrendExp.exp_id IS NULL"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *newExp = p_psDBFetchResult(config->dbh); + if (!output) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "failed to fetch the database result"); + return false; + } + + for (long i = 0; i < psArrayLength(newExp); i++) { + newExpRow *exp = newExp->data[i]; + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", 0, NULL, exp->exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + psFree(newExp); + return NULL; + } + new + + psFree(where); + } + return new; } +#endif static p1PendingExpRow *newToP1PendingExp(newExpRow *newExp) @@ -379,4 +484,5 @@ } +#if 0 static rawScienceFrame *newToRawScienceFrame(pxConfig *config, newFrame *newFrame) { @@ -411,4 +517,6 @@ return rawScienceFrame; } + +#endif static rawScienceExpRow *newToRawScienceExp(pxConfig *config, newExpRow *exp) @@ -488,3 +596,213 @@ } - +static rawDetrendExpRow *newToRawDetrendExp(pxConfig *config, newExpRow *exp) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(exp, NULL); + + bool status = false; + psString filter = psMetadataLookupStr(&status, config->args, "-filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter"); + return false; + } + if (!filter) { + psError(PS_ERR_UNKNOWN, true, "-filter is required"); + return false; + } + psF32 airmass = psMetadataLookupF32(&status, config->args, "-airmass"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass"); + return false; + } + if (isnan(airmass)) { + psError(PS_ERR_UNKNOWN, true, "-airmass is required"); + return false; + } + psF64 ra = psMetadataLookupF64(&status, config->args, "-ra"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ra"); + return false; + } + if (isnan(ra)) { + psError(PS_ERR_UNKNOWN, true, "-airmass is required"); + return false; + } + psF64 decl = psMetadataLookupF64(&status, config->args, "-decl"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -decl"); + return false; + } + if (isnan(decl)) { + psError(PS_ERR_UNKNOWN, true, "-decl is required"); + return false; + } + psF32 exp_time = psMetadataLookupF32(&status, config->args, "-exp_time"); + if (isnan(exp_time)) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time"); + return false; + } + if (!exp_time) { + psError(PS_ERR_UNKNOWN, true, "-exp_time is required"); + return false; + } + psF64 background = psMetadataLookupF64(&status, config->args, "-background"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -background"); + return false; + } + if (!exp_time) { + psError(PS_ERR_UNKNOWN, true, "-background is required"); + return false; + } + + return rawDetrendExpRowAlloc( + exp->exp_id, + exp->camera, + exp->telescope, + exp->exp_type, + exp->imfiles, + filter, + airmass, + ra, + decl, + exp_time, + background + ); +} + +static rawImfileRow *newToRawImfile(pxConfig *config, newImfileRow *imfile) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(exp, NULL); + + bool status = false; + psString filter = psMetadataLookupStr(&status, config->args, "-filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter"); + return false; + } + if (!filter) { + psError(PS_ERR_UNKNOWN, true, "-filter is required"); + return false; + } + psF32 airmass = psMetadataLookupF32(&status, config->args, "-airmass"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass"); + return false; + } + if (isnan(airmass)) { + psError(PS_ERR_UNKNOWN, true, "-airmass is required"); + return false; + } + psF64 ra = psMetadataLookupF64(&status, config->args, "-ra"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ra"); + return false; + } + if (isnan(ra)) { + psError(PS_ERR_UNKNOWN, true, "-ra is required"); + return false; + } + psF64 decl = psMetadataLookupF64(&status, config->args, "-decl"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -decl"); + return false; + } + if (isnan(decl)) { + psError(PS_ERR_UNKNOWN, true, "-decl is required"); + return false; + } + psF32 exp_time = psMetadataLookupF32(&status, config->args, "-exp_time"); + if (isnan(exp_time)) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time"); + return false; + } + if (isnan(exp_time)) { + psError(PS_ERR_UNKNOWN, true, "-exp_time is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + if (isnan(bg)) { + psError(PS_ERR_UNKNOWN, true, "-bg is required"); + return false; + } + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + if (isnan(bg_stdev)) { + psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + return false; + } + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + if (isnan(bg_mean_stdev)) { + psError(PS_ERR_UNKNOWN, true, "-bg_mean_stdev is required"); + return false; + } + psF64 alt = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -alt"); + return false; + } + if (isnan(alt)) { + psError(PS_ERR_UNKNOWN, true, "-alt is required"); + return false; + } + psF64 az = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -az"); + return false; + } + if (isnan(az)) { + psError(PS_ERR_UNKNOWN, true, "-az is required"); + return false; + } + psF32 ccd_temp = psMetadataLookupF32(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp"); + return false; + } + if (isnan(ccd_temp)) { + psError(PS_ERR_UNKNOWN, true, "-ccd_temp is required"); + return false; + } + psF64 posang = psMetadataLookupF32(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang"); + return false; + } + if (isnan(posang)) { + psError(PS_ERR_UNKNOWN, true, "-posang is required"); + return false; + } + + return rawImfileRowAlloc( + imfile->exp_id, + imfile->class, + imfile->class_id, + imfile->uri, + filter, + airmass, + ra, + decl, + exp_time, + bg, + bg_stdev, + bg_mean_stdev, + alt, + az, + ccd_temp, + posang + ); +} +