Index: /trunk/ippTools/scripts/chiptest.sh =================================================================== --- /trunk/ippTools/scripts/chiptest.sh (revision 12173) +++ /trunk/ippTools/scripts/chiptest.sh (revision 12174) @@ -8,9 +8,9 @@ for ID in `seq 0 3`; do - chiptool -addprocessedimfile -chip_id 1 -class_id $ID -uri file://chipp-t10.$ID -recip myrecipe -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file:///foo || exit 1 + chiptool -addprocessedimfile -chip_id 1 -class_id $ID -uri file://chipp-t10.$ID -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file:///foo || exit 1 done; for ID in `seq 0 3`; do - chiptool -addprocessedimfile -chip_id 2 -class_id $ID -uri file://chipp-t11.$ID -recip myrecipe -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file:///foo || exit 1 + chiptool -addprocessedimfile -chip_id 2 -class_id $ID -uri file://chipp-t11.$ID -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file:///foo || exit 1 done; Index: /trunk/ippTools/share/Makefile.am =================================================================== --- /trunk/ippTools/share/Makefile.am (revision 12173) +++ /trunk/ippTools/share/Makefile.am (revision 12174) @@ -9,3 +9,4 @@ regtool_find_unprocessed_imfile.sql \ regtool_processedimfile.sql \ - regtool_pendingexp.sql + regtool_pendingexp.sql \ + chiptool_queuerawexp.sql Index: /trunk/ippTools/share/chiptool_completely_processed_exp.sql =================================================================== --- /trunk/ippTools/share/chiptool_completely_processed_exp.sql (revision 12173) +++ /trunk/ippTools/share/chiptool_completely_processed_exp.sql (revision 12174) @@ -12,14 +12,14 @@ ON chipPendingExp.exp_tag = rawExp.exp_tag LEFT JOIN chipProcessedExp - ON chipPendingExp.exp_tag = chipProcessedExp.exp_tag + ON chipPendingExp.chip_id = chipProcessedExp.chip_id LEFT JOIN chipPendingImfile - ON chipPendingExp.exp_tag = chipPendingImfile.exp_tag + ON chipPendingExp.chip_id = chipPendingImfile.chip_id LEFT JOIN chipProcessedImfile - ON chipPendingExp.exp_tag = chipProcessedImfile.exp_tag + ON chipPendingExp.chip_id = chipProcessedImfile.chip_id WHERE - chipProcessedExp.exp_tag IS NULL - AND chipPendingImfile.exp_tag IS NULL - AND chipProcessedImfile.exp_tag IS NOT NULL + chipProcessedExp.chip_id IS NULL + AND chipPendingImfile.chip_id IS NULL + AND chipProcessedImfile.chip_id IS NOT NULL GROUP BY - chipPendingExp.exp_tag + chipPendingExp.chip_id HAVING rawExp.imfiles = COUNT(chipProcessedImfile.class_id) Index: /trunk/ippTools/share/chiptool_find_unprocessed_imfile.pl =================================================================== --- /trunk/ippTools/share/chiptool_find_unprocessed_imfile.pl (revision 12173) +++ /trunk/ippTools/share/chiptool_find_unprocessed_imfile.pl (revision 12174) @@ -3,6 +3,6 @@ FROM chipPendingImfile LEFT JOIN chipProcessedImfile - USING(exp_tag, class_id) + USING(chip_id, class_id) WHERE - chipProcessedImfile.exp_tag IS NULL + chipProcessedImfile.chip_id IS NULL AND chipProcessedImfile.class_id IS NULL Index: /trunk/ippTools/share/chiptool_pendingimfile.sql =================================================================== --- /trunk/ippTools/share/chiptool_pendingimfile.sql (revision 12173) +++ /trunk/ippTools/share/chiptool_pendingimfile.sql (revision 12174) @@ -5,7 +5,7 @@ FROM chipPendingImfile JOIN chipPendingExp - USING(exp_tag) + USING(chip_id) JOIN rawExp - USING(exp_tag) + ON chipPendingExp.exp_tag = rawExp.exp_tag LEFT JOIN chipMask ON chipPendingExp.label = chipMask.label Index: /trunk/ippTools/share/chiptool_queuerawexp.sql =================================================================== --- /trunk/ippTools/share/chiptool_queuerawexp.sql (revision 12174) +++ /trunk/ippTools/share/chiptool_queuerawexp.sql (revision 12174) @@ -0,0 +1,15 @@ +-- we can only enqueue one exp_tag at a time or the generated chip_id of all +-- put the last one inserted will be lost +INSERT INTO chipPendingExp + SElECT + 0, -- chip_id + exp_tag, -- exp_tag + 0xdeadbeef, -- guide_id + '%s', -- label + '%s', -- recipe + '%s', -- expgroup + '%s' -- dvodb + FROM rawExp + WHERE + rawExp.fault = 0 + AND rawExp.exp_tag = '%s' Index: /trunk/ippTools/share/chiptool_queuerawimfile.sql =================================================================== --- /trunk/ippTools/share/chiptool_queuerawimfile.sql (revision 12174) +++ /trunk/ippTools/share/chiptool_queuerawimfile.sql (revision 12174) @@ -0,0 +1,11 @@ +INSERT INTO chipPendingImfile + SELECT + chipPendingExp.chip_id, + rawImfile.class_id, + rawImfile.uri + FROM chipPendingExp + JOIN rawImfile + USING(exp_tag) + WHERE + rawImfile.fault = 0 + AND chipPendingExp.chip_id = LAST_INSERT_ID() Index: /trunk/ippTools/src/chiptool.c =================================================================== --- /trunk/ippTools/src/chiptool.c (revision 12173) +++ /trunk/ippTools/src/chiptool.c (revision 12174) @@ -111,5 +111,5 @@ { \ bool status = false; \ - ps##type var = psMetadataLookup##type(&status, from, "-" #name); \ + ps##type var = psMetadataLookup##type(&status, from, "-" name); \ if (!status) { \ psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -" name); \ @@ -118,5 +118,5 @@ if (!isnan(var)) { \ if (!psMetadataAdd##type(to, PS_LIST_TAIL, #name, 0, "==", var)) { \ - psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " name); \ psFree(to); \ return false; \ @@ -153,7 +153,7 @@ ADDPARAMSTR(config->args, where, "object"); - if (config->where->list->n < 1) { - psFree(config->where); - config->where = NULL; + if (where->list->n < 1) { + psFree(where); + where = NULL; } @@ -165,28 +165,33 @@ } - // XXX this does not work!!! + psString recipe = psMetadataLookupStr(&status, config->args, "-set_recipe"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_recipe"); + return false; + } + + psString expgroup = psMetadataLookupStr(&status, config->args, "-set_expgroup"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_expgroup"); + return false; + } + + psString dvodb = psMetadataLookupStr(&status, config->args, "-set_dvodb"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_dvodb"); + return false; + } + + // find the exp_tag of all the exposures that we want to queue up. psString query = psStringCopy( - "INSERT INTO chipPendingExp\n" - " SElECT\n" - " 0\n" // chip_id - " exp_tag,\n" - " 'my recipe',\n" // recipe - " 255\n" // guide_version - " 255\n" // chip_version - " %s\n" - " FROM rawExp" - " LEFT JOIN chipProcessedExp" - " USING(exp_tag)" - " WHERE" - " rawExp.fault = 0" - ); - - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawExp"); + "SELECT exp_tag FROM rawExp WHERE rawExp.fault = 0"); + if (where) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); + psFree(where); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } - if (!p_psDBRunQuery(config->dbh, query, label ? label : "NULL")) { + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -201,7 +206,104 @@ } if (!psArrayLength(output)) { - // XXX check psError here - psError(PS_ERR_UNKNOWN, false, "no chipPendingImfile rows found"); - psFree(output); + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // load the SQL to enqueue our exp_tags from disk once + psString queuerawexp_query = pxDataGet("chiptool_queuerawexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(output); + return false; + } + + psString queuerawimfile_query = pxDataGet("chiptool_queuerawimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(queuerawexp_query); + psFree(output); + return false; + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(queuerawimfile_query); + psFree(queuerawexp_query); + psFree(output); + return false; + } + + // loop over our list of exp_tags + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + psString exp_tag = psMetadataLookupStr(&status, md, "exp_tag"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_tag"); + psFree(queuerawimfile_query); + psFree(queuerawexp_query); + psFree(output); + return false; + } + + // queue the exp + if (!p_psDBRunQuery(config->dbh, queuerawexp_query, + label ? label : "NULL", + recipe ? recipe : "NULL", + expgroup ? expgroup : "NULL", + dvodb ? dvodb : "NULL", + exp_tag + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(queuerawimfile_query); + psFree(queuerawexp_query); + psFree(output); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row"); + psFree(queuerawimfile_query); + psFree(queuerawexp_query); + psFree(output); + return false; + } + + // queue the imfiles for the exp we just queued + if (!p_psDBRunQuery(config->dbh, queuerawimfile_query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(queuerawimfile_query); + psFree(queuerawexp_query); + psFree(output); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row"); + psFree(queuerawimfile_query); + psFree(queuerawexp_query); + psFree(output); + return false; + } + } + psFree(queuerawimfile_query); + psFree(queuerawexp_query); + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); return false; } @@ -776,14 +878,4 @@ } - psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); - return false; - } - if (!recipe) { - psError(PS_ERR_UNKNOWN, true, "-recip is required"); - return false; - } - psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); if (!status) { @@ -820,8 +912,5 @@ return chipProcessedImfileRowAlloc( imfile->chip_id, - imfile->exp_tag, - imfile->guide_id, imfile->class_id, - recipe, uri, bg, Index: /trunk/ippTools/src/chiptoolConfig.c =================================================================== --- /trunk/ippTools/src/chiptoolConfig.c (revision 12173) +++ /trunk/ippTools/src/chiptoolConfig.c (revision 12174) @@ -86,5 +86,11 @@ "search by exposure object", NULL); psMetadataAddStr(queuerawexpArgs, PS_LIST_TAIL, "-set_label", 0, - "define label for phase 2 processing (non-detrend data only)", NULL); + "define label", NULL); + psMetadataAddStr(queuerawexpArgs, PS_LIST_TAIL, "-set_recipe", 0, + "define recipe", NULL); + psMetadataAddStr(queuerawexpArgs, PS_LIST_TAIL, "-set_expgroup", 0, + "define exposure group", NULL); + psMetadataAddStr(queuerawexpArgs, PS_LIST_TAIL, "-set_dvodb", 0, + "define DBO db", NULL); @@ -114,14 +120,8 @@ psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "define chip ID (required)", NULL); -#if 0 - psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-exp_tag", 0, - "define exposure tag (required) (XXX redudant with chip_id)", NULL); -#endif psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID", NULL); psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-uri", 0, "define URL", NULL); - psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-recip", 0, - "define recipe (required)", NULL); psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposue background", NAN); Index: /trunk/ippTools/src/regtool.c =================================================================== --- /trunk/ippTools/src/regtool.c (revision 12173) +++ /trunk/ippTools/src/regtool.c (revision 12174) @@ -910,8 +910,5 @@ return chipPendingImfileRowAlloc( chip_id, - rawImfile->exp_tag, - 0xdeadbeef, // guide version rawImfile->class_id, - "my recipe", // recipe rawImfile->uri );