Index: trunk/ippTools/src/chiptool.c =================================================================== --- trunk/ippTools/src/chiptool.c (revision 29994) +++ trunk/ippTools/src/chiptool.c (revision 30049) @@ -271,7 +271,8 @@ // default + PXOPT_LOOKUP_BOOL(unique, config->args, "-unique", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - + // find the exp_id of all the exposures that we want to queue up. psString query = pxDataGet("chiptool_find_rawexp.sql"); @@ -288,4 +289,8 @@ psFree(where); + if (unique) { + psStringAppend(&query, "AND chip_id IS NULL"); + } + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: trunk/ippTools/src/chiptoolConfig.c =================================================================== --- trunk/ippTools/src/chiptoolConfig.c (revision 29994) +++ trunk/ippTools/src/chiptoolConfig.c (revision 30049) @@ -61,4 +61,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define distribution group", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-unique", 0, "only queue exposures that have no previous chipRun", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); Index: trunk/ippTools/src/pztool.c =================================================================== --- trunk/ippTools/src/pztool.c (revision 29994) +++ trunk/ippTools/src/pztool.c (revision 30049) @@ -112,5 +112,5 @@ PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); - + if (!pzDataStoreInsert(config->dbh, camera, Index: trunk/ippTools/src/regtool.c =================================================================== --- trunk/ippTools/src/regtool.c (revision 29994) +++ trunk/ippTools/src/regtool.c (revision 30049) @@ -34,4 +34,6 @@ static bool pendingimfileMode(pxConfig *config); static bool addprocessedimfileMode(pxConfig *config); +static bool checkburntoolimfileMode(pxConfig *config); +static bool pendingburntoolimfileMode(pxConfig *config); static bool processedimfileMode(pxConfig *config); static bool revertprocessedimfileMode(pxConfig *config); @@ -72,4 +74,6 @@ // imfile MODECASE(REGTOOL_MODE_PENDINGIMFILE, pendingimfileMode); + MODECASE(REGTOOL_MODE_CHECKBURNTOOLIMFILE, checkburntoolimfileMode); + MODECASE(REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileMode); MODECASE(REGTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileMode); MODECASE(REGTOOL_MODE_PROCESSEDIMFILE, processedimfileMode); @@ -176,4 +180,289 @@ } +static bool checkburntoolimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config,false); + + // required + PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(date, config->args, "-date", true, false); + PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false); + // optional + PXOPT_LOOKUP_STR(camera, config->args, "-inst", false, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + psString query = pxDataGet("regtool_checkburntoolimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + psString rep = psStringCopy(query); + psFree(query); + query = rep; + + psStringSubstitute(&query,exp_name,"@EXP_NAME@"); + psStringSubstitute(&query,class_id,"@CLASS_ID@"); + psStringSubstitute(&query,date,"@DATE@"); + + fprintf(stderr,"%s",query); + + if (!p_psDBRunQuery(config->dbh, query)) { + // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms) + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + psString previous_uri = NULL; + bool ok_to_burn = true; + bool already_burned = true; + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; + + if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)|| + (psMetadataLookupS32(NULL,row,"is_registered") != 1)) { + ok_to_burn = false; + } + if (already_burned == false) { + ok_to_burn = false; + } + if (abs(psMetadataLookupS32(NULL,row,"burntool_state")) == valid_burntool) { + already_burned = true; + } + else { + already_burned = false; + } + + if (previous_uri) { + psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri); + psFree(previous_uri); + } + psMetadataAddBool(row,PS_LIST_TAIL,"burnable",PS_META_REPLACE,"",ok_to_burn); + psMetadataAddBool(row,PS_LIST_TAIL,"already_burned",PS_META_REPLACE,"",already_burned); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "regBurntoolImfile", !simple)) { + psError(PXTOOLS_ERR_PROG, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + + return(true); + +} + +static bool pendingburntoolimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config,false); + + // required + PXOPT_LOOKUP_STR(date, config->args, "-date", true, false); + PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false); + // optional + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(ignore_state, config->args, "-ignore_state", false); + psString query = pxDataGet("regtool_pendingburntoolimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + psString rep = psStringCopy(query); + psFree(query); + query = rep; + + psStringSubstitute(&query,date,"@DATE@"); + + fprintf(stderr,"%s",query); + + if (!p_psDBRunQuery(config->dbh, query)) { + // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms) + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + psString previous_uri = NULL; + psString this_uri = NULL; + bool ok_to_burn = true; + bool already_burned = true; + psString previous_class_id = NULL; + psString this_class_id = NULL; + psArray *results = psArrayAllocEmpty(60); // List of suggested imfiles to burntool. + + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; + + // Add the information about this row and the previous, if it exists. + // Write the class_id stuff for debugging. + this_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); + psStringSubstitute(&this_class_id,"XY","ota"); + psMetadataAddStr(row,PS_LIST_TAIL,"this_class_id",PS_META_REPLACE,"",this_class_id); + if (previous_class_id) { + psMetadataAddStr(row,PS_LIST_TAIL,"previous_class_id",PS_META_REPLACE,"",previous_class_id); + } + + // Determine if we've crossed a class_id boundary, as this resets the bits. + if (previous_class_id) { + if (strcmp(this_class_id,previous_class_id) != 0) { + ok_to_burn = true; + already_burned = true; + previous_uri = NULL; + } + } + + // Write the URIs as well. + this_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Duplicate, but helpful for my debugging. + psMetadataAddStr(row,PS_LIST_TAIL,"this_uri",PS_META_REPLACE,"",this_uri); + if (previous_uri) { + psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri); + } + + // Check the two status variables + // Convert bits of the SQL query into booleans describing the data state + if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)|| + (psMetadataLookupS32(NULL,row,"is_registered") != 1)) { + ok_to_burn = false; + } + if (already_burned == false) { + ok_to_burn = false; + } + if (abs(psMetadataLookupS32(NULL,row,"burntool_state")) == valid_burntool) { + already_burned = true; + } + else { + already_burned = false; + } + psMetadataAddBool(row,PS_LIST_TAIL,"burnable",PS_META_REPLACE,"",ok_to_burn); + psMetadataAddBool(row,PS_LIST_TAIL,"already_burned",PS_META_REPLACE,"",already_burned); + + // Check the uri for this exposure + if (!this_uri) { + ok_to_burn = false; + already_burned = false; + + // Save this round for next round. + psFree(previous_class_id); + psFree(previous_uri); + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); + psStringSubstitute(&previous_class_id,"XY","ota"); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + continue; + } + + // printf("STATUS: %s %s %s %s (%d %d)\n",this_uri,previous_uri,this_class_id,previous_class_id,ok_to_burn,already_burned); + + // If the state of this imfile is not "pending_burntool" then we can't burn it. + if (!ignore_state) { + psString imfile_state = psMetadataLookupStr(NULL,row,"imfile_state"); + + if (!imfile_state) { // imfile state is NULL, so we probably aren't registered. + ok_to_burn = false; + + // Save this round for next round. + psFree(previous_class_id); + psFree(previous_uri); + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); + psStringSubstitute(&previous_class_id,"XY","ota"); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + continue; + } + if (strcmp("pending_burntool",imfile_state) != 0) { // Probably the state is full, do not twiddle states + // Save this round for next round. + psFree(previous_class_id); + psFree(previous_uri); + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); + psStringSubstitute(&previous_class_id,"XY","ota"); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + continue; + } + } + + // Determine if we've already suggested an entry for this ota, and if not, copy this + // suggestion into our output result list. + if (!ok_to_burn || already_burned) { + // Save this round for next round. + psFree(previous_class_id); + psFree(previous_uri); + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); + psStringSubstitute(&previous_class_id,"XY","ota"); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + continue; + } + // If we're here, then we think we could potentially burntool this file. + psArrayAdd(results,results->n,row); + + // Save this round for next round. + psFree(previous_class_id); + psFree(previous_uri); + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); + psStringSubstitute(&previous_class_id,"XY","ota"); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, results, "regPendingBurntoolImfile", !simple)) { + psError(PXTOOLS_ERR_PROG, false, "failed to print array"); + psFree(output); + psFree(results); + return false; + } + + psFree(output); + psFree(results); + + + return(true); + +} + + static bool addprocessedimfileMode(pxConfig *config) @@ -250,5 +539,6 @@ PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false); PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); - + PXOPT_LOOKUP_BOOL(video_cells, config->args, "-video_cells", false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); @@ -323,4 +613,5 @@ bytes, md5sum, + video_cells, 0 // burntool_state )) { @@ -809,4 +1100,7 @@ PXOPT_LOOKUP_F32(moon_phase, config->args, "-moon_phase", false, false); PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false); + PXOPT_LOOKUP_STR(chip_workdir,config->args, "-chip_workdir", false, false); PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); @@ -1008,13 +1302,13 @@ // else continue on... - + chip_workdir = chip_workdir ? chip_workdir : workdir; // insert an entry into the chipPendingExp table // this can only be run as the newExp's state has been set to stop if (!pxchipQueueByExpTag(config, exp_id, - workdir, + chip_workdir, label, - NULL, // data_group - NULL, // dist_group + data_group, + dist_group, reduction, NULL, // expgroup @@ -1092,5 +1386,7 @@ PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); - PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_STR(config->args, where, "-object", "object", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "obs_mode", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-comment", "comment", "LIKE"); PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "sun_angle", ">="); PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); Index: trunk/ippTools/src/regtool.h =================================================================== --- trunk/ippTools/src/regtool.h (revision 29994) +++ trunk/ippTools/src/regtool.h (revision 30049) @@ -26,4 +26,6 @@ REGTOOL_MODE_NONE = 0x0, REGTOOL_MODE_PENDINGIMFILE, + REGTOOL_MODE_CHECKBURNTOOLIMFILE, + REGTOOL_MODE_PENDINGBURNTOOLIMFILE, REGTOOL_MODE_ADDPROCESSEDIMFILE, REGTOOL_MODE_PROCESSEDIMFILE, Index: trunk/ippTools/src/regtoolConfig.c =================================================================== --- trunk/ippTools/src/regtoolConfig.c (revision 29994) +++ trunk/ippTools/src/regtoolConfig.c (revision 30049) @@ -53,4 +53,22 @@ ADD_OPT(Bool, pendingimfileArgs, "-simple", "use the simple output format", false); + // -checkburntoolimfile + psMetadata *checkburntoolimfileArgs = psMetadataAlloc(); + ADD_OPT(Str, checkburntoolimfileArgs, "-exp_name", "define the exp_name (required)", NULL); + ADD_OPT(Str, checkburntoolimfileArgs, "-class_id", "define class ID (required)", NULL); + ADD_OPT(Str, checkburntoolimfileArgs, "-date", "select the date to process (required)", NULL); + ADD_OPT(S32, checkburntoolimfileArgs, "-valid_burntool", "define the good burntool value (required)", 0); + ADD_OPT(Str, checkburntoolimfileArgs, "-inst", "define the camera name", NULL); + ADD_OPT(Str, checkburntoolimfileArgs, "-telescope", "define the telescope name", NULL); + ADD_OPT(Bool, checkburntoolimfileArgs, "-simple", "use the simple output format", false); + + // -pendingburntoolimfile + psMetadata *pendingburntoolimfileArgs = psMetadataAlloc(); + ADD_OPT(Str, pendingburntoolimfileArgs, "-date", "select the date to process (required)", NULL); + ADD_OPT(S32, pendingburntoolimfileArgs, "-valid_burntool", "define the good burntool value (required)", 0); + ADD_OPT(Bool, pendingburntoolimfileArgs, "-simple", "use the simple output format", false); + ADD_OPT(Bool, pendingburntoolimfileArgs, "-ignore_state", "ignore the data_state when deciding what to work on", false); + ADD_OPT(U64, pendingburntoolimfileArgs, "-limit", "limit result set to N items", 0); + // -addprocessedimfile psMetadata *addprocessedimfileArgs = psMetadataAlloc(); @@ -128,4 +146,5 @@ ADD_OPT(S32, addprocessedimfileArgs, "-bytes", "define bytes", 0); ADD_OPT(S16, addprocessedimfileArgs, "-burntool_state", "set burntool state", 0); + ADD_OPT(Bool, addprocessedimfileArgs, "-video_cells", "define if chip has video cells", false); ADD_OPT(S16, addprocessedimfileArgs, "-fault", "set fault code", 0); ADD_OPT(S16, addprocessedimfileArgs, "-quality", "set quality flag", 0); @@ -249,4 +268,7 @@ ADD_OPT(F32, addprocessedexpArgs, "-moon_phase", "define moon phase (0.0 = new)", NAN); ADD_OPT(Str, addprocessedexpArgs, "-label", "define label for chip stage (non-detrend data only)", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-data_group", "define data_group for chip stage (non-detrend data only)", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-dist_group", "define dist_group for chip stage (non-detrend data only)", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-chip_workdir", "define workdir for chip stage (non-detrend data only)", NULL); ADD_OPT(Str, addprocessedexpArgs, "-hostname", "define host name", NULL); ADD_OPT(S16, addprocessedexpArgs, "-fault", "set fault code", 0); @@ -291,4 +313,7 @@ psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by exposure obs_mode", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-comment", 0, "search by exposure comment", NULL); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); @@ -395,4 +420,6 @@ PXOPT_ADD_MODE("-pendingimfile", "", REGTOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + PXOPT_ADD_MODE("-checkburntoolimfile", "", REGTOOL_MODE_CHECKBURNTOOLIMFILE, checkburntoolimfileArgs); + PXOPT_ADD_MODE("-pendingburntoolimfile", "", REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileArgs); PXOPT_ADD_MODE("-addprocessedimfile", "", REGTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); PXOPT_ADD_MODE("-processedimfile", "", REGTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs);