Index: branches/eam_branches/ipp-20111122/ippTools/src/Makefile.am =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/Makefile.am (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/Makefile.am (revision 33638) @@ -29,4 +29,5 @@ diffphottool \ minidvodbtool \ + mergetool \ laptool \ vptool @@ -43,4 +44,5 @@ pxmagic.h \ pxminidvodb.h \ + pxmerge.h \ pxregister.h \ pxspace.h \ @@ -79,4 +81,5 @@ diffphottool.h \ minidvodbtool.h \ + mergetool.h \ laptool.h \ vptool.h @@ -98,4 +101,5 @@ pxmagic.c \ pxminidvodb.c \ + pxmerge.c \ pxregister.c \ pxspace.c \ @@ -294,4 +298,10 @@ minidvodbtoolConfig.c +mergetool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +mergetool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +mergetool_SOURCES = \ + mergetool.c \ + mergetoolConfig.c + laptool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) laptool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la Index: branches/eam_branches/ipp-20111122/ippTools/src/addtool.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/addtool.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/addtool.c (revision 33638) @@ -141,11 +141,4 @@ pxAddLabelSearchArgs (config, where, "-data_group","staticskyRun.data_group", "=="); // define using camRun label PXOPT_COPY_STR(config->args, where, "-reduction", "staticskyyRun.reduction", "=="); - pxAddLabelSearchArgs (config, where, "-filter", "stackRun.filter", "=="); // define using camRun label - } - if (strcmp(stage, "staticsky_multi")== 0) { - - pxAddLabelSearchArgs (config, where, "-label", "staticskyRun.label", "=="); // define using camRun label - pxAddLabelSearchArgs (config, where, "-data_group","staticskyRun.data_group", "=="); // define using camRun label - PXOPT_COPY_STR(config->args, where, "-reduction", "staticskyyRun.reduction", "=="); //no filter here } @@ -227,21 +220,6 @@ } } + if (strcmp(stage,"staticsky") == 0) { - if (dvodb ) { - psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_sky_id_dvo.sql\n%s\n", dvodb,stage); - // find the cam_id of all the exposures that we want to queue up. - bare_query = pxDataGet("addtool_find_sky_id_dvo.sql"); - // user supplied dvodb - psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); - } else { - psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_sky_id.sql\n%s\n",stage); - // find the cam_id of all the exposures that we want to queue up. - bare_query = pxDataGet("addtool_find_sky_id.sql"); - // inherit dvodb from camRun, avoid matching NULL - psStringAppend(&dvodb_string, "(staticskyRun.dvodb IS NOT NULL AND previous_dvodb = staticskyRun.dvodb)"); - } - } - - if (strcmp(stage,"staticsky_multi") == 0) { if (dvodb ) { psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_sky_id_multi_dvo.sql\n%s\n", dvodb,stage); @@ -292,7 +270,8 @@ } else { //This picks only the unmagicked/uncensored ones - if (strcmp(stage,"cam") == 0) { - psStringAppend(&query, " AND (camRun.magicked = 0)"); - } + //if (strcmp(stage,"cam") == 0) { + //we can now properly handle the magicked case in ippScripts, so we queue camRuns in any magic state now if -uncensored. + //psStringAppend(&query, " AND (camRun.magicked = 0)"); + //} if (strcmp(stage,"stack") == 0) { psStringAppend(&query, " AND (stackRun.magicked = 0)"); @@ -309,8 +288,8 @@ psStringAppend(&query, " GROUP BY stack_id"); } + // if (strcmp(stage,"staticsky") == 0) { + // psStringAppend(&query, " GROUP BY stack_id"); + // } if (strcmp(stage,"staticsky") == 0) { - psStringAppend(&query, " GROUP BY stack_id"); - } - if (strcmp(stage,"staticsky_multi") == 0) { psStringAppend(&query, " GROUP BY sky_id"); //some reason it needs this } @@ -401,31 +380,6 @@ } } + if (strcmp(stage,"staticsky") == 0) { - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - - staticskyRunRow *row = staticskyRunObjectFromMetadata(md); - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); - psFree(output); - return false; - } - - if (!dvodb) { //there's no staticsky.dvodb - psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, sky_id %" PRId64, row->label, row->sky_id); - psFree(output); - return false; - } - if (!workdir && !row->workdir) { - psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, sky_id %" PRId64, row->label, row->sky_id); - psFree(output); - return false; - } - - psFree(row); - } - } - if (strcmp(stage,"staticsky_multi") == 0) { for (long i = 0; i < psArrayLength(output); i++) { psMetadata *md = output->data[i]; @@ -565,48 +519,6 @@ } } + if (strcmp(stage,"staticsky") == 0) { - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - psS64 stage_id =0; - - staticskyRunRow *row = staticskyRunObjectFromMetadata(md); - stage_id = row->sky_id; - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); - psFree(output); - return false; - } - - // queue the exp - if (!pxaddQueueByCamID(config, - stage, - stage_id, - 0, - workdir ? workdir : row->workdir, - reduction ? reduction : row->reduction, - label ? label : row->label, - data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), - dvodb ? dvodb : NULL, - note ? note : NULL, - image_only, - minidvodb, - minidvodb_group, - minidvodb_name - )) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error sfg"); - } - psError(PS_ERR_UNKNOWN, false, - "failed to trying to queue stage %s %" PRId64,stage, stage_id); - psFree(row); - psFree(output); - return false; - } - psFree(row); - } - } - - if (strcmp(stage,"staticsky_multi") == 0) { for (long i = 0; i < psArrayLength(output); i++) { psMetadata *md = output->data[i]; @@ -704,7 +616,5 @@ query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id"); } - if (strcmp(stage, "staticsky_multi")==0) { - query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id"); - } + // pxUpdateRun gets parameters from config->args and runs the update query @@ -744,7 +654,4 @@ } if (strcmp(stage, "staticsky")==0) { - query = pxDataGet("addtool_find_pendingexp_staticsky.sql"); - } - if (strcmp(stage, "staticsky_multi")==0) { query = pxDataGet("addtool_find_pendingexp_staticsky_multi.sql"); } @@ -767,9 +674,9 @@ psStringAppend(&query, " GROUP BY %s", "stack_id"); } + //if (strcmp(stage, "staticsky") == 0) { + // //this group by is needed to join against all the warps (to get camera) + // psStringAppend(&query, " GROUP BY %s", "sky_id"); + //} if (strcmp(stage, "staticsky") == 0) { - //this group by is needed to join against all the warps (to get camera) - psStringAppend(&query, " GROUP BY %s", "sky_id"); - } - if (strcmp(stage, "staticsky_multi") == 0) { //this group by is needed to join against all the warps (to get camera) psStringAppend(&query, " GROUP BY %s", "sky_id, stage_extra1"); @@ -824,5 +731,5 @@ PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-minidvodb_name", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - + PXOPT_LOOKUP_S64(stage_extra1, config->args, "-stage_extra1", false, false); // generate restrictions psMetadata *where = psMetadataAlloc(); @@ -902,5 +809,19 @@ } } - + //if there is a stage_extra1, set it in addRun (it's not known until it is processed) + if (stage_extra1) { + psString setExtra = NULL; + psStringAppend (&setExtra, "UPDATE addRun set stage_extra1 = %" PRId64, stage_extra1); + psStringAppend (&setExtra, " where add_id = %" PRId64, row->add_id); + + if (!p_psDBRunQuery(config->dbh, setExtra)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + + return false; + } + } @@ -962,7 +883,4 @@ } if (strcmp (stage,"staticsky") == 0) { - query = pxDataGet("addtool_find_processedexp_staticsky.sql"); - } - if (strcmp (stage,"staticsky_multi") == 0) { query = pxDataGet("addtool_find_processedexp_staticsky_multi.sql"); } @@ -1049,5 +967,5 @@ PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); PXOPT_COPY_S64(config->args, where, "-stage_id", "addRun.stage_id", "=="); - PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); pxcamGetSearchArgs (config, where); pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); @@ -1075,8 +993,5 @@ query = pxDataGet("addtool_revertprocessedexp_stack.sql"); } - if (strcmp(stage, "staticsky") == 0) { - query = pxDataGet("addtool_revertprocessedexp_staticsky.sql"); - } - if (strcmp(stage, "staticsky_multi") == 0) { + if (strcmp(stage, "staticsky") == 0) { query = pxDataGet("addtool_revertprocessedexp_staticsky_multi.sql"); } @@ -1221,7 +1136,5 @@ PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", true, false); - PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", true, false); - - //optional + //optional PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-set_minidvodb_name", false, false); PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false); @@ -1238,4 +1151,7 @@ if (minidvodb_path) { minidvodbpath = minidvodb_path; + } else { + psError(PS_ERR_UNKNOWN, false, "require minidvodb_path"); + return false; } @@ -1245,6 +1161,5 @@ minidvodb_group, minidvodbpath, - mergedvodb_path, - "new", + "new", 0 )) { @@ -1268,9 +1183,4 @@ psStringAppend(&minidvodb_name, "%s.%" PRIu64,minidvodb_group,minidvodb_id); } - - if (!minidvodb_path) { - psStringAppend(&minidvodb_path,"%s/%s",mergedvodb_path,minidvodb_name); - } - if (minidvodb_path) { psStringAppend(&minidvodb_path,"/%s",minidvodb_name); @@ -1304,5 +1214,4 @@ PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); PXOPT_COPY_STR(config->args, where, "-minidvodb_path", "minidvodb_path", "=="); - PXOPT_COPY_STR(config->args, where, "-mergedvodb_path", "mergedvodb_path", "=="); PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodb_group", "=="); @@ -1310,5 +1219,4 @@ PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", false, false); PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", false, false); @@ -1345,12 +1253,4 @@ } - if (mergedvodb_path) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - psStringAppend(&query, " mergedvodb_path = '%s'", mergedvodb_path); - cnt++; - } - if (minidvodb_group) { if (cnt) { @@ -1396,5 +1296,5 @@ } - psString firstquery = psStringCopy("SELECT * from minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL"); + psString firstquery = psStringCopy("SELECT * from minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL"); psString firstwhereClause = psDBGenerateWhereConditionSQL(where, NULL); @@ -1444,5 +1344,5 @@ //now flip new -> active - psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL "); + psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL "); psStringAppend(&query2, " AND minidvodb_group = '%s' limit 1;", minidvodb_group); if (!p_psDBRunQuery(config->dbh, query2)) { @@ -1480,6 +1380,6 @@ PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbRun.minidvodb_name", "=="); PXOPT_COPY_STR(config->args, where, "-state", "minidvodbRun.state", "=="); - - + PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-minidvodb_group", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(all_addrun_states, config->args, "-all_addrun_states", false); @@ -1493,18 +1393,33 @@ } - psString query = pxDataGet("addtool_checkminidvodbaddrun.sql"); - - if (!query) { + if (!minidvodb_group) { + psError(PXTOOLS_ERR_CONFIG, false, "minidvodb_group required"); + return false; + } + + + psString bare_query = pxDataGet("addtool_checkminidvodbaddrun.sql"); + + if (!bare_query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } - + psString where_string = NULL; if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&where_string, " WHERE %s", whereClause); psFree(whereClause); - } - + //search params are defined above + } + + psString query = NULL; + psStringAppend(&query, bare_query, minidvodb_group); + psStringAppend(&query, where_string); + + + + + if (!all_addrun_states) { psStringAppend(&query, " AND (cnt2 = cnt) "); @@ -1519,5 +1434,5 @@ if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psError(PS_ERR_UNKNOWN, false, "database error,\n\n%s", query); psFree(query); return false; @@ -1621,13 +1536,9 @@ // required PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-minidvodb_id", true, false); - PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-mergedvodb_path", true, false); PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-minidvodb_group", true, false); // optional - PXOPT_LOOKUP_U64(merge_order, config->args, "-merge_order", false, false); PXOPT_LOOKUP_F32(dtime_relphot, config->args, "-dtime_relphot", false, false); PXOPT_LOOKUP_F32(dtime_resort, config->args, "-dtime_resort", false, false); - PXOPT_LOOKUP_F32(dtime_merge, config->args, "-dtime_merge", false, false); - PXOPT_LOOKUP_F32(dtime_verify, config->args, "-dtime_verify", false, false); PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); @@ -1680,12 +1591,8 @@ minidvodbProcessedRow *row = minidvodbProcessedRowAlloc( pendingRow->minidvodb_id, - merge_order, dtime_resort, dtime_relphot, - dtime_verify, - dtime_merge, dtime_script, epoch, - mergedvodb_path, fault ); @@ -1702,67 +1609,4 @@ } - - - - //this finds the # of merged things (for the merge order) - psString query3 = NULL; - psStringAppend(&query3, "select count(*) from minidvodbRun join minidvodbProcessed using (minidvodb_id) where state = 'merged' and minidvodb_group = '%s';", minidvodb_group); - - if (!p_psDBRunQuery(config->dbh, query3)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query3); - return false; - } - psArray *output2 = p_psDBFetchResult(config->dbh); - if (!output2) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output2)) { - psTrace("addtool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - bool status; - psS64 m_order = psMetadataLookupS64(&status, output2->data[0], "count(*)"); - if (!status) { - - psAbort("failed to lookup value for count column"); - return false; - } - psString final = NULL; - psStringAppend(&final, "%" PRIu64, m_order); - //return false; - psFree(query3); - psFree(output2); - - - - //update the merge_order - - psString query4 = NULL; - psStringAppend(&query4, "update minidvodbProcessed set merge_order = %"PRIu64,m_order); - psStringAppend(&query4," where minidvodb_id = %" PRIu64, minidvodb_id); - //printf("%s", query4); - if (!p_psDBRunQuery(config->dbh, query4)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query4); - return false; - } - // -psFree(query4); - - - -// since there is only one exp per 'new' set mindvodbRun.state = 'merged' - psString query2 = NULL ; psStringAppend(&query2, "UPDATE minidvodbRun SET state = 'merged' WHERE minidvodb_id = %'" PRIu64, row->minidvodb_id); @@ -1777,12 +1621,4 @@ return false; } - - - - - - - - psFree(row); psFree(pendingRow); @@ -1797,10 +1633,4 @@ return false; } - - - //print the merge_order (why not!) - printf("%s", final); - psFree(final); - return true; } @@ -1961,10 +1791,7 @@ PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-minidvodb_id", true, false); - PXOPT_LOOKUP_U64(merge_order, config->args, "-set_merge_order", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); PXOPT_LOOKUP_F32(dtime_relphot, config->args, "-set_dtime_relphot", false, false); PXOPT_LOOKUP_F32(dtime_resort, config->args, "-set_dtime_resort", false, false); - PXOPT_LOOKUP_F32(dtime_merge, config->args, "-set_dtime_merge", false, false); -PXOPT_LOOKUP_F32(dtime_verify, config->args, "-set_dtime_verify", false, false); PXOPT_LOOKUP_F32(dtime_script, config->args, "-set_dtime_script", false, false); PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbProcessed.minidvodb_id", "=="); @@ -1986,13 +1813,4 @@ } - if (merge_order) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - - psStringAppend(&query, " merge_order = %" PRId64, merge_order); - cnt++; - } - if (dtime_relphot) { if (cnt) { @@ -2011,19 +1829,5 @@ } - if (dtime_merge) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - psStringAppend(&query, " dtime_merge = %f", dtime_merge); - cnt++; - } -if (dtime_verify) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - psStringAppend(&query, " dtime_verify = %f", dtime_verify); - cnt++; - } -if (dtime_script) { + if (dtime_script) { if (cnt) { psStringAppend(&query, "%s", comma); Index: branches/eam_branches/ipp-20111122/ippTools/src/addtoolConfig.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/addtoolConfig.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/addtoolConfig.c (revision 33638) @@ -115,4 +115,6 @@ psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN); psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); +psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-stage_extra1", 0, "set stage_extra1", 0); + // -processedexp @@ -165,5 +167,4 @@ psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group", 0, "define minidvodb_group (required)", NULL); psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path", 0, "define path for minidvodb", NULL); - psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define path for the merged dvodb (required)", NULL); psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); @@ -174,9 +175,7 @@ psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "search by minidvodb_name (LIKE)", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_path", 0, "search by path for minidvodb", NULL); - psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-mergedvodb_path", 0, "search by path for the merged dvodb", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_name", 0, "define minidvodb_name", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path", 0, "define path for minidvodb", NULL); - psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define path for minidvodb", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group", 0, "define path for the merged dvodb", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); @@ -205,12 +204,8 @@ psMetadata *addminidvodbprocessedArgs = psMetadataAlloc(); psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "define minidvodb_id (required)", 0); - psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-merge_order", 0, "define merge order", 0); psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_relphot", 0, "define elapsed time for relphot (seconds)", NAN); psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_resort", 0, "define elapsed time for resort (seconds)", NAN); - psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_verify", 0, "define elapsed time for DVO verify (seconds)", NAN); - psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_merge", 0, "define elapsed time for DVO merge (seconds)", NAN); psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time for script (seconds)", NAN); psMetadataAddTime(addminidvodbprocessedArgs, PS_LIST_TAIL, "-epoch", 0, "time merge is finished", NULL); - psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-mergedvodb_path",0, "path of merged dvodb", NULL); psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_group",0, "minidvodb_group", NULL); psMetadataAddS16(addminidvodbprocessedArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); @@ -232,9 +227,6 @@ psMetadataAddStr(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "search by minidvodb_name", NULL); psMetadataAddS16(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); - psMetadataAddU64(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_merge_order", 0, "define merge order", 0); psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_relphot", 0, "define elapsed time for relphot (seconds)", 0); psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_resort", 0, "define elapsed time for resort (seconds)", 0); - psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_verify", 0, "define elapsed time for DVO verify (seconds)", 0); - psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_merge", 0, "define elapsed time for DVO merge (seconds)", 0); psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_script", 0, "define elapsed time for script (seconds)", 0); psMetadata *argSets = psMetadataAlloc(); Index: branches/eam_branches/ipp-20111122/ippTools/src/camtool.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/camtool.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/camtool.c (revision 33638) @@ -112,4 +112,5 @@ PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); PXOPT_COPY_STR(config->args, where, "-data_group", "chipRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "=="); if (!psListLength(where->list) && Index: branches/eam_branches/ipp-20111122/ippTools/src/camtoolConfig.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/camtoolConfig.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/camtoolConfig.c (revision 33638) @@ -55,4 +55,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by chipRun reduction class", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-data_group", 0, "search by chipRun data_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by rawExp obs_mode", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); Index: branches/eam_branches/ipp-20111122/ippTools/src/chiptool.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/chiptool.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/chiptool.c (revision 33638) @@ -198,4 +198,11 @@ GET_VALUE (psS64, S64, exp_id, "exp_id"); GET_VALUE (psString, Str, raw_workdir, "workdir"); +#ifdef notyet + GET_VALUE (psString, Str, old_workdir, "old_workdir"); + GET_VALUE (psString, Str, old_data_group, "old_data_group"); +#else + const char *old_workdir = NULL; + const char *old_data_group = NULL; +#endif GET_VALUE (psString, Str, raw_label, "label"); GET_VALUE (psString, Str, raw_reduction, "reduction"); @@ -213,7 +220,7 @@ if (!pxchipQueueByExpTag(config, exp_id, - workdir ? workdir : raw_workdir, + workdir ? workdir : (old_workdir ? old_workdir : raw_workdir), label ? label : raw_label, - data_group ? data_group : (label ? label : raw_label), + data_group ? data_group : (old_data_group ? old_data_group : (label ? label : raw_label)), dist_group, reduction ? reduction : raw_reduction, @@ -345,4 +352,5 @@ pxAddLabelSearchArgs (config, where, "-label", "chipOld.label", "LIKE"); pxAddLabelSearchArgs (config, where, "-data_group", "chipOld.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-state", "chipOld.state", "=="); // psListLength(where->list) is at least 1 because exp_type defaults to "object" @@ -354,5 +362,5 @@ } - PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); @@ -468,4 +476,5 @@ PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "rawImfile.class_id", "=="); psString query = pxDataGet("chiptool_pendingimfile.sql"); @@ -1006,5 +1015,9 @@ PXOPT_COPY_S64(config->args, where, "-chip_id", "chipProcessedImfile.chip_id", "=="); PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "=="); - PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + + PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", false, false); + // if cam_id is not supplied chip_id is required + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", cam_id ? false : true, false); PXOPT_LOOKUP_S64(class_id, config->args, "-class_id", true, false); PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", true, false); @@ -1017,5 +1030,5 @@ if (psListLength(where->list) == 0) { // won't get here - psError(PXTOOLS_ERR_CONFIG, true, "-chip_id and -class_id are required"); + psError(PXTOOLS_ERR_CONFIG, true, "-chip_id (or -cam_id) and -class_id are required"); return false; } @@ -2024,7 +2037,16 @@ return false; } - psFree(setHook); psFree(query); + + // if we are updating a whole chipRun set lingering chips with poor quality to full data_state + if (!class_id) { + query = "UPDATE chipProcessedImfile SET data_state ='full', fault = 0 WHERE chip_id = %" PRId64 " AND quality != 0 AND (data_state ='cleaned' OR data_state = 'update')"; + + if (!p_psDBRunQueryF(config->dbh, query, chip_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } return true; Index: branches/eam_branches/ipp-20111122/ippTools/src/chiptoolConfig.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/chiptoolConfig.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/chiptoolConfig.c (revision 33638) @@ -72,6 +72,7 @@ psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-exp_type", PS_META_REPLACE, "search by exp_type", "object"); - - psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-state", 0, "search by chipRun.state", NULL); + + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); @@ -111,4 +112,5 @@ psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id", 0); pxchipSetSearchArgs(pendingimfileArgs); psMetadataAddU64(pendingimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -240,5 +242,6 @@ // -dropprocessedimfile psMetadata *dropprocessedimfileArgs = psMetadataAlloc(); - psMetadataAddS64(dropprocessedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID (required)", 0); + psMetadataAddS64(dropprocessedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddS64(dropprocessedimfileArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam ID", 0); psMetadataAddStr(dropprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID (required)", NULL); psMetadataAddS16(dropprocessedimfileArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality (required)", 0); Index: branches/eam_branches/ipp-20111122/ippTools/src/difftool.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/difftool.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/difftool.c (revision 33638) @@ -1981,5 +1981,5 @@ psString finishQuery = NULL; psStringAppend( &finishQuery, - "UPDATE diffRun set state ='full', note = 'dummy run - no overlap' WHERE diff_id = %" PRId64, + "UPDATE diffRun set state ='full', dist_group = NULL, note = 'dummy run - no overlap' WHERE diff_id = %" PRId64, run->diff_id); if (!p_psDBRunQuery(config->dbh, finishQuery)) { Index: branches/eam_branches/ipp-20111122/ippTools/src/disttool.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/disttool.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/disttool.c (revision 33638) @@ -34,7 +34,9 @@ static bool updaterunMode(pxConfig *config); static bool revertrunMode(pxConfig *config); -static bool rerunMode(pxConfig *config); +static bool startoverMode(pxConfig *config); static bool pendingcomponentMode(pxConfig *config); static bool addprocessedcomponentMode(pxConfig *config); +static bool updateprocessedcomponentMode(pxConfig *config); +static bool revertcomponentMode(pxConfig *config); static bool revertcomponentMode(pxConfig *config); static bool processedcomponentMode(pxConfig *config); @@ -84,7 +86,8 @@ MODECASE(DISTTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(DISTTOOL_MODE_REVERTRUN, revertrunMode); - MODECASE(DISTTOOL_MODE_RERUN, rerunMode); + MODECASE(DISTTOOL_MODE_STARTOVER, startoverMode); MODECASE(DISTTOOL_MODE_PENDINGCOMPONENT, pendingcomponentMode); MODECASE(DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentMode); + MODECASE(DISTTOOL_MODE_UPDATEPROCESSEDCOMPONENT, updateprocessedcomponentMode); MODECASE(DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentMode); MODECASE(DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentMode); @@ -165,4 +168,5 @@ PXOPT_LOOKUP_BOOL(single, config->args, "-singlefilter", false); + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); if (use_alternate) { @@ -404,4 +408,7 @@ psStringAppend(&query, " AND (camRun.state = 'full')"); } + } + if (!rerun) { + psStringAppend(&query, " AND (distRun.dist_id IS NULL)"); } @@ -643,5 +650,5 @@ } -static bool rerunMode(pxConfig *config) +static bool startoverMode(pxConfig *config) { psMetadata *where = psMetadataAlloc(); @@ -653,5 +660,5 @@ // require data_group or dist_id to be supplied - PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", true, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", false, false); PXOPT_LOOKUP_S64(dist_id, config->args, "-dist_id", false, false); if (!data_group && !dist_id) { @@ -969,4 +976,60 @@ return true; } +static bool updateprocessedcomponentMode(pxConfig *config) +{ + + // required values + PXOPT_LOOKUP_S64(dist_id, config->args, "-dist_id", true, false); + PXOPT_LOOKUP_STR(component, config->args, "-component", true, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false); + PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false); + PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); + PXOPT_LOOKUP_STR(outdir, config->args, "-outdir", false, false); + PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); + + bool setfault = clearfault || fault; + + if (!bytes && !md5sum && !outdir && !name && !setfault && !fault) { + psError(PS_ERR_UNKNOWN, true, "at least one of bytes md5sum outdir name fault or setfault is required"); + return false; + } + + char *sep = ""; + psString query = psStringCopy("UPDATE distComponent SET "); + if (setfault) { + psStringAppend(&query, "%s fault = %d", sep, fault ? 1 : 0); + sep = ", "; + } + if (bytes) { + psStringAppend(&query, "%s bytes = %d", sep, bytes); + sep = ", "; + } + if (md5sum) { + psStringAppend(&query, "%s md5sum = '%s'", sep, md5sum); + sep = ", "; + } + if (outdir) { + psStringAppend(&query, "%s outdir = '%s'", sep, outdir); + sep = ", "; + } + if (name) { + psStringAppend(&query, "%s name = '%s'", sep, name); + sep = ", "; + } + + psStringAppend(&query, "\nWHERE dist_id = %"PRId64 " AND component = '%s'", + dist_id, component); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} static bool toadvanceMode(pxConfig *config) @@ -1048,4 +1111,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); + PXOPT_COPY_STR(config->args, where, "-component", "component", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -2185,4 +2249,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-fs_id", "fs_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); if (!psListLength(where->list)) { Index: branches/eam_branches/ipp-20111122/ippTools/src/disttool.h =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/disttool.h (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/disttool.h (revision 33638) @@ -28,9 +28,10 @@ DISTTOOL_MODE_UPDATERUN, DISTTOOL_MODE_REVERTRUN, - DISTTOOL_MODE_RERUN, + DISTTOOL_MODE_STARTOVER, DISTTOOL_MODE_PENDINGCOMPONENT, DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, DISTTOOL_MODE_REVERTCOMPONENT, DISTTOOL_MODE_PROCESSEDCOMPONENT, + DISTTOOL_MODE_UPDATEPROCESSEDCOMPONENT, DISTTOOL_MODE_TOADVANCE, DISTTOOL_MODE_PENDINGCLEANUP, Index: branches/eam_branches/ipp-20111122/ippTools/src/disttoolConfig.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/disttoolConfig.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/disttoolConfig.c (revision 33638) @@ -65,4 +65,5 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-singlefilter", 0, "select single filter runs (sky stage only)", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue a new distRun even if one already exists", false); psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -87,15 +88,15 @@ psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); - // -rerun - psMetadata *rerunArgs = psMetadataAlloc(); - psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-data_group", 0, "define data_group (required)", NULL); - psMetadataAddS64(rerunArgs, PS_LIST_TAIL, "-dist_id", 0, "search by dist_id", 0); - psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-stage", 0, "search by stage", NULL); - psMetadataAddS64(rerunArgs, PS_LIST_TAIL, "-stage_id", 0, "search by stage_id", 0); - psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); - psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-set_label", 0, "define new label", NULL); - psMetadataAddBool(rerunArgs, PS_LIST_TAIL, "-pretend", 0, "don't queue runs just display what would be selected", false); - psMetadataAddU64(rerunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(rerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -startover + psMetadata *startoverArgs = psMetadataAlloc(); + psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-data_group", 0, "define data_group", NULL); + psMetadataAddS64(startoverArgs, PS_LIST_TAIL, "-dist_id", 0, "search by dist_id", 0); + psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-stage", 0, "search by stage", NULL); + psMetadataAddS64(startoverArgs, PS_LIST_TAIL, "-stage_id", 0, "search by stage_id", 0); + psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-set_label", 0, "define new label", NULL); + psMetadataAddBool(startoverArgs, PS_LIST_TAIL, "-pretend", 0, "don't queue runs just display what would be selected", false); + psMetadataAddU64(startoverArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(startoverArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -revertrun @@ -151,6 +152,18 @@ psMetadata *processedcomponentArgs = psMetadataAlloc(); psMetadataAddS64(processedcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); + psMetadataAddStr(processedcomponentArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL); psMetadataAddU64(processedcomponentArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(processedcomponentArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updateprocessedcomponentArgs + psMetadata *updateprocessedcomponentArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); + psMetadataAddStr(updateprocessedcomponentArgs, PS_LIST_TAIL, "-component", 0, "define component (required)", NULL); + psMetadataAddStr(updateprocessedcomponentArgs, PS_LIST_TAIL, "-outdir", 0, "define output directory", NULL); + psMetadataAddStr(updateprocessedcomponentArgs, PS_LIST_TAIL, "-name", 0, "define file name", NULL); + psMetadataAddS32(updateprocessedcomponentArgs, PS_LIST_TAIL, "-bytes", 0, "define file size", 0); + psMetadataAddStr(updateprocessedcomponentArgs, PS_LIST_TAIL, "-md5sum", 0, "define stage for bundle", NULL); + psMetadataAddS32(updateprocessedcomponentArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + psMetadataAddBool(updateprocessedcomponentArgs, PS_LIST_TAIL, "-clearfault", 0, "set fault to zero", false); // -toadvance @@ -180,5 +193,5 @@ psMetadata *updatefilesetArgs = psMetadataAlloc(); psMetadataAddS64(updatefilesetArgs, PS_LIST_TAIL, "-fs_id", 0, "define fs_id", 0); -// psMetadataAddS64(updatefilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); + psMetadataAddS64(updatefilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); psMetadataAddStr(updatefilesetArgs, PS_LIST_TAIL, "-set_state",0, "new value for state", NULL); psMetadataAddS32(updatefilesetArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); @@ -343,8 +356,9 @@ PXOPT_ADD_MODE("-definebyquery", "define distribution run", DISTTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); PXOPT_ADD_MODE("-updaterun", "update distRun", DISTTOOL_MODE_UPDATERUN, updaterunArgs); - PXOPT_ADD_MODE("-rerun", "reprocess a cleaned distribution run", DISTTOOL_MODE_RERUN, rerunArgs); + PXOPT_ADD_MODE("-startover", "reprocess a cleaned distribution run", DISTTOOL_MODE_STARTOVER, startoverArgs); PXOPT_ADD_MODE("-revertrun", "revert distRun.faults", DISTTOOL_MODE_REVERTRUN, revertrunArgs); PXOPT_ADD_MODE("-pendingcomponent", "", DISTTOOL_MODE_PENDINGCOMPONENT, pendingcomponentArgs); PXOPT_ADD_MODE("-addprocessedcomponent", "", DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentArgs); + PXOPT_ADD_MODE("-updateprocessedcomponent", "", DISTTOOL_MODE_UPDATEPROCESSEDCOMPONENT, updateprocessedcomponentArgs); PXOPT_ADD_MODE("-revertcomponent", "revert faulted components", DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentArgs); PXOPT_ADD_MODE("-processedcomponent", "", DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentArgs); Index: branches/eam_branches/ipp-20111122/ippTools/src/faketool.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/faketool.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/faketool.c (revision 33638) @@ -335,4 +335,5 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-label", "fakeRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "fakeRun.state", "=="); PXOPT_COPY_STR(config->args, where, "-data_group", "fakeRun.data_group", "=="); PXOPT_COPY_STR(config->args, where, "-dist_group", "fakeRun.dist_group", "=="); Index: branches/eam_branches/ipp-20111122/ippTools/src/laptool.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/laptool.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/laptool.c (revision 33638) @@ -390,4 +390,5 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); psMetadata *where = psMetadataAlloc(); @@ -411,4 +412,9 @@ psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); + } + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n %s", limitString); + psFree(limitString); } Index: branches/eam_branches/ipp-20111122/ippTools/src/laptoolConfig.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/laptoolConfig.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/laptoolConfig.c (revision 33638) @@ -74,4 +74,5 @@ ADD_OPT(Str, pendingrunArgs, "-fault", "search by LAP run fault", NULL); ADD_OPT(Bool,pendingrunArgs, "-simple", "use the simple output format", false); + ADD_OPT(U64, pendingrunArgs, "-limit", "limit result set to N items", 0); // -listrun @@ -86,4 +87,5 @@ ADD_OPT(Str, listrunArgs, "-fault", "search by LAP run fault", NULL); ADD_OPT(Bool,listrunArgs, "-simple", "use the simple output format", false); + ADD_OPT(U64, listrunArgs, "-limit", "limit result set to N items", 0); // -updaterun Index: branches/eam_branches/ipp-20111122/ippTools/src/magicdstool.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/magicdstool.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/magicdstool.c (revision 33638) @@ -779,5 +779,5 @@ case IPP_STAGE_CAMERA: query = NULL; - clearRunQuery = "UPDATE chipBackgroundRun SET magicked = 0 where chip_bg_id = %" PRId64; + clearRunQuery = "UPDATE camRun SET magicked = 0 where cam_id = %" PRId64; break; case IPP_STAGE_WARP: @@ -1242,4 +1242,6 @@ } else if (!strcmp(state, "failed_cleanup")) { new_state = "goto_cleaned"; + } else if (!strcmp(state, "failed_restore")) { + new_state = "goto_restored"; } else { psError(PS_ERR_UNKNOWN, true, "unexpected value for state: %s", state); @@ -1410,4 +1412,5 @@ (strcmp(state, "drop") == 0) || (strcmp(state, "wait") == 0) || + (strcmp(state, "failed_restore") == 0) || (strcmp(state, "failed_revert") == 0) || (strcmp(state, "failed_revert_ud") == 0) || Index: branches/eam_branches/ipp-20111122/ippTools/src/mergetool.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/mergetool.c (revision 33638) +++ branches/eam_branches/ipp-20111122/ippTools/src/mergetool.c (revision 33638) @@ -0,0 +1,1227 @@ +/* + * addtool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxmerge.h" +#include "mergetool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool pendingmergeMode(pxConfig *config); +static bool addmergedMode(pxConfig *config); +static bool listmergedMode(pxConfig *config); +static bool revertmergedMode(pxConfig *config); +static bool updatemergedMode(pxConfig *config); +static bool definebyquerymergecopyMode(pxConfig *config); +static bool listmergedvodbcopyMode(pxConfig *config); +static bool revertmergedvodbcopyMode(pxConfig *config); +static bool updatemergedvodbcopyMode(pxConfig *config); + + + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = mergetoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(MERGETOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(MERGETOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(MERGETOOL_MODE_PENDINGMERGE, pendingmergeMode); + MODECASE(MERGETOOL_MODE_ADDMERGED, addmergedMode); + MODECASE(MERGETOOL_MODE_LISTMERGED, listmergedMode); + MODECASE(MERGETOOL_MODE_REVERTMERGED, revertmergedMode); + MODECASE(MERGETOOL_MODE_UPDATEMERGED, updatemergedMode); + MODECASE(MERGETOOL_MODE_DEFINEBYQUERYMERGECOPY, definebyquerymergecopyMode); + MODECASE(MERGETOOL_MODE_LISTMERGEDVODBCOPY, listmergedvodbcopyMode); + MODECASE(MERGETOOL_MODE_REVERTMERGEDVODBCOPY, revertmergedvodbcopyMode); + MODECASE(MERGETOOL_MODE_UPDATEMERGEDVODBCOPY, updatemergedvodbcopyMode); + + + + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodb_id", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "=="); + // PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", false, false); + PXOPT_LOOKUP_STR(mergedvodb, config->args, "-mergedvodb", true, false); + + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + //find the items to queue (uniq minidvodb_id) + psString dvodb_string = NULL; + psString bare_query = NULL; + +bare_query = pxDataGet("mergetool_find_minidvodb_id.sql"); + psStringAppend(&dvodb_string, "mergedvodbRun.mergedvodb = '%s'", mergedvodb); + +if (!bare_query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + psFree(where); + return false; + } + // Take the bare query and add the dvodb selector + psString query = NULL; + psStringAppend(&query, bare_query, dvodb_string); + psFree(dvodb_string); + psFree(bare_query); + + // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; + } + psFree(where); + + + psStringAppend(&query, " GROUP BY minidvodb_id"); + + + psTrace("mergetool.c", PS_LOG_INFO,"query: \n\n%s\n\n",query); + +if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error, \n%s\n", query); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + +for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + minidvodbRunRow *row = minidvodbRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into minidvodbRun"); + psFree(output); + return false; + } + + if (!mergedvodb) { + psError(PS_ERR_UNKNOWN, false, "cannot queue merge run without a defined mergedvodb: minidvodb_id %" PRId64,row->minidvodb_id); + psFree(output); + return false; + } + if (!mergedvodb_path) { + psError(PS_ERR_UNKNOWN, false, "cannot queue merge run without a defined mergedvodb_path: minidvodb_id %" PRId64,row->minidvodb_id); + psFree(output); + return false; + } + + psFree(row); + +} + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } +for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 minidvodb_id =0; + + minidvodbRunRow *row = minidvodbRunObjectFromMetadata(md); + minidvodb_id = row->minidvodb_id; + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into minidvodbRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxmergeQueueByMinidvodbID(config, + minidvodb_id, + mergedvodb, + mergedvodb_path + + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue mergedvodb %" PRId64, minidvodb_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + + } + +psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + + + + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", false, false); + psString query = NULL; + + query = psStringCopy("UPDATE mergedvodbRun SET"); + + int cnt = 0; + psString comma = ","; + if (state) { + psStringAppend(&query, " state = '%s'", state); + cnt++; + } + + if (mergedvodb_path) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " mergedvodb_path = '%s'", mergedvodb_path); + cnt++; + } + + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + + return true; +} + +static bool pendingmergeMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "mergedvodbRun.minidvodb_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = NULL; + query = pxDataGet("mergetool_find_pending_merge.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "mergedvodbPendingMerge", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addmergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_LOOKUP_U64(merge_id, config->args, "-merge_id", true, false); + PXOPT_LOOKUP_STR(mergedvodb, config->args, "-mergedvodb", true, false); + + // optional + +//PXOPT_LOOKUP_U64(merge_order, config->args, "-merge_order", false, false); + PXOPT_LOOKUP_F32(dtime_merge, config->args, "-dtime_merge", false, false); + PXOPT_LOOKUP_F32(dtime_verify, config->args, "-dtime_verify", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + + PXOPT_LOOKUP_TIME(epoch, config->args, "-epoch", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psString query = pxDataGet("mergetool_find_pendingmergeprocess.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClaus = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClaus); + psFree(whereClaus); + } + psFree(where); + psTrace("mergetool.c", PS_LOG_INFO, "query is:\n%s", query); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + +psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("mergetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + mergedvodbRunRow *pendingRow = mergedvodbRunObjectFromMetadata(output->data[0]); + psFree(output); + mergedvodbProcessedRow *row = mergedvodbProcessedRowAlloc( + pendingRow->merge_id, + 0, + dtime_verify ? dtime_verify : 0, + dtime_merge ? dtime_merge : 0, + dtime_script ? dtime_script : 0, + epoch, + fault + ); + + if (!mergedvodbProcessedInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + psFree(pendingRow); + return false; + } + +//this finds the # of merged things (for the merge order) + psString query3 = NULL; + psStringAppend(&query3, "select count(*) from mergedvodbRun join mergedvodbProcessed using (merge_id) where state = 'full' and mergedvodb = '%s';", mergedvodb); + + if (!p_psDBRunQuery(config->dbh, query3)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query3); + return false; + } + psArray *output2 = p_psDBFetchResult(config->dbh); + if (!output2) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output2)) { + psTrace("mergetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + bool status; + psS64 m_order = psMetadataLookupS64(&status, output2->data[0], "count(*)"); + if (!status) { + + psAbort("failed to lookup value for count column"); + return false; + } + psString final = NULL; + psStringAppend(&final, "%" PRIu64, m_order); + //return false; + psFree(query3); + psFree(output2); + + + + //update the merge_order + + psString query4 = NULL; + psStringAppend(&query4, "update mergedvodbProcessed set merge_order = %"PRIu64,m_order); + psStringAppend(&query4," where merge_id = %" PRIu64, merge_id); + //printf("%s", query4); + if (!p_psDBRunQuery(config->dbh, query4)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query4); + return false; + } + // +psFree(query4); + +// since there is only one exp per 'new' set mergedvodbRun.state = 'full' + + psString query2 = NULL ; + + if (!fault) { + psStringAppend(&query2, "UPDATE mergedvodbRun SET state = 'full' WHERE merge_id = %'" PRIu64, row->merge_id); + + if (!p_psDBRunQuery(config->dbh, query2)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query2); + return false; + } + + } + + + + + + + psFree(row); + psFree(pendingRow); + + + + + + //commit the changes + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + + + //print the merge_order (why not!) + printf("%s", final); + psFree(final); + + + + + + + + + + + + return true; +} + + +static bool listmergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "mergedvodbRun.minidvodb_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psString query = NULL; + query = pxDataGet("mergetool_find_processed.sql"); + +if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + + + // we either add AND (condition) or WHERE (condition): + if (psListLength(where->list) && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " AND fault != 0"); + } + if (psListLength(where->list) && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND fault = 0"); + } + if (!psListLength(where->list) && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " WHERE fault != 0"); + } + if (!psListLength(where->list) && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " WHERE fault = 0"); + } + +psFree(where); + + // order by merge_id + psStringAppend(&query, " ORDER BY merge_id"); + + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + + + return true; +} + + +static bool revertmergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "mergedvodbRun.fault", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = NULL; + query = pxDataGet("mergetool_revertmergedvodbprocessed.sql"); + +if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + + return true; +} + +static bool updatemergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-merge_id", true, false); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbProcessed.merge_id", "=="); + PXOPT_LOOKUP_F32(dtime_merge, config->args, "-set_dtime_merge", false, false); + PXOPT_LOOKUP_F32(dtime_verify, config->args, "-set_dtime_verify", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-set_dtime_script", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psString query = psStringCopy("UPDATE mergedvodbProcessed JOIN mergedvodbRun USING (merge_id) SET "); +int cnt = 0; + psString comma = ","; + if (fault) { + psStringAppend(&query, " fault = %d", fault); + cnt++; + } + + + + if (dtime_verify) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_verify = %f", dtime_verify); + cnt++; + } + + if (dtime_merge) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_merge = %f", dtime_merge); + cnt++; + } + +if (dtime_script) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_script = %f", dtime_script); + cnt++; + } +if (state) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " state = '%s'", state); + cnt++; + } + + + + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error \n%s\n",query ); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + return true; +} + +static bool definebyquerymergecopyMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + pxAddLabelSearchArgs (config, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); // define using camRun label + + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(destination_host, config->args, "-set_destination_host", false, false); + PXOPT_LOOKUP_STR(mergedvodb_rsync_path, config->args, "-set_mergedvodb_rsync_path", false, false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(last_merged, config->args, "-last_merged", false); + // prevent queueing an addRun if a given exposure has already been added to + // the given dvo database + psString dvodb_string = NULL; + psString bare_query = NULL; + if (destination_host) { + psTrace("mergedvodbtool.c", PS_LOG_INFO, "destination_host argument found (%s) using mergedvodbtool_find_merge_id_dvo.sql\n", destination_host); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("mergedvodbtool_find_merge_id_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "mergedvodbCopy.destination_host = '%s'", destination_host); + } else { + psError(PS_ERR_UNKNOWN, false, "cannot queue mergedvodbcopy run without a defined destination_host"); + + return false; + } + + if (!bare_query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + psFree(where); + return false; + } + // Take the bare query and add the dvodb selector + psString query = NULL; + psStringAppend(&query, bare_query, dvodb_string); + psFree(dvodb_string); + psFree(bare_query); + + // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; + } + if (last_merged) { + psStringAppend(&query, " ORDER BY merge_order DESC LIMIT 1"); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "mergedvodbCopy", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + // loop over our list of camRun rows to check the supplied and selected dvodb and workdir values: + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + mergedvodbRunRow *row = mergedvodbRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into mergedvodbRun"); + psFree(output); + return false; + } + + if (!mergedvodb_rsync_path) { + psError(PS_ERR_UNKNOWN, false, "cannot queue mergedvodbcopy run without a defined mergedvodb_rsync_path: merge_id %" PRId64, row->merge_id); + psFree(output); + return false; + } + if (!destination_host) { + psError(PS_ERR_UNKNOWN, false, "cannot queue mergedvodbcopy run without a defined destination_host: merge_id %" PRId64, row->merge_id); + psFree(output); + return false; + } + + psFree(row); + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + mergedvodbRunRow *row = mergedvodbRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByMergeID(config, + row->merge_id, + destination_host ? destination_host : "NULL", + mergedvodb_rsync_path ?mergedvodb_rsync_path : "NULL" + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue merge_id: %" PRId64, row->merge_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool listmergedvodbcopyMode(pxConfig *config) { + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbCopy.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodbcopy_id", "mergedvodbCopy.mergedvodbcopy_id", "=="); + PXOPT_COPY_STR(config->args, where, "-destination_host", "mergedvodbCopy.destination_host", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pending, config->args, "-pending", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("mergedvodbtool_find_mergedvodbcopy.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + +if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + +// we either add AND (condition) or WHERE (condition): + if (where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault != 0"); + } + if (where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault = 0"); + } + if (!where->list && faulted) { + + // list only faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault != 0"); + } + if (where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault = 0"); + } + if (!where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " WHERE mergedvodbCopy.fault != 0"); + } + if (!where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " WHERE mergedvodbCopy.fault = 0"); + } + psFree(where); + + if (pending) { + //add the cuts for pending (state new, no faults) + psStringAppend(&query, " %s", " AND mergedvodbCopy.state = 'new' AND mergedvodbCopy.fault = 0 AND mergedvodbCopy.destination_host IS NOT NULL AND mergedvodbCopy.mergedvodb_rsync_path IS NOT NULL"); + } + + + // order by epoch + psStringAppend(&query, " ORDER BY mergedvodbcopy_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error %s ", query); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "mergedvodbCopy", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + +return true; +} + +static bool revertmergedvodbcopyMode(pxConfig *config) { + psMetadata *where = psMetadataAlloc(); + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_COPY_S64(config->args, where, "-mergedvodbcopy_id", "mergedvodbCopy.mergedvodbcopy_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbCopy.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-destination_host", "destination_host", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "mergedvodbCopy.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = pxDataGet("mergedvodbtool_revertmergedvodbcopy.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + } + psError(PS_ERR_UNKNOWN, false, "database error %s", query ); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool updatemergedvodbcopyMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_U64(mergedvodbcopy_id, config->args, "-mergedvodbcopy_id", false, false); + PXOPT_LOOKUP_U64(merge_id, config->args, "-merge_id", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + PXOPT_LOOKUP_STR(host, config->args, "-host", false, false); + PXOPT_LOOKUP_STR(rsync_path, config->args, "-mergedvodb_rsync_path", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + PXOPT_LOOKUP_S16(set_fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_STR(set_rsync_path, config->args, "-set_mergedvodb_rsync_path", false, false); + PXOPT_LOOKUP_STR(set_host, config->args, "-set_destination_host", false, false); + PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_F32(dtime, config->args, "-set_dtime", false, false); + PXOPT_COPY_S64(config->args, where, "-mergedvodbcopy_id", "mergedvodbCopy.mergedvodbcopy_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbCopy.merge_id", "=="); + + PXOPT_COPY_STR(config->args, where, "-state", "mergedvodbCopy.state", "=="); + PXOPT_COPY_STR(config->args, where, "-host", "mergedvodbCopy.destination_host", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb_rsync_path", "mergedvodbCopy.mergedvodb_rsync_path", "=="); + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE mergedvodbCopy JOIN mergedvodbRun USING (merge_id) SET "); + int cnt = 0; + psString comma = ","; + if (set_fault) { + psStringAppend(&query, " mergedvodbCopy.fault = %d", set_fault); + cnt++; + } + if (set_rsync_path) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " mergedvodbCopy.mergedvodb_rsync_path = '%s'" , set_rsync_path); + cnt++; + } + + if (set_host) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " mergedvodbCopy.destination_host = '%s'" , set_host); + cnt++; + } + + + if (set_state) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " mergedvodbCopy.state = '%s'" , set_state); + cnt++; + } + + if (dtime) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " mergedvodbCopy.dtime = %f", dtime); + cnt++; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + return true; +} + + + Index: branches/eam_branches/ipp-20111122/ippTools/src/mergetool.h =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/mergetool.h (revision 33638) +++ branches/eam_branches/ipp-20111122/ippTools/src/mergetool.h (revision 33638) @@ -0,0 +1,41 @@ +/* + * mergetool.h + * + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef MERGETOOL_H +#define MERGETOOL_H 1 + +#include "pxtools.h" + +typedef enum { + MERGETOOL_MODE_NONE = 0x0, + MERGETOOL_MODE_DEFINEBYQUERY, + MERGETOOL_MODE_UPDATERUN, + MERGETOOL_MODE_PENDINGMERGE, + MERGETOOL_MODE_ADDMERGED, + MERGETOOL_MODE_LISTMERGED, + MERGETOOL_MODE_REVERTMERGED, + MERGETOOL_MODE_UPDATEMERGED, + MERGETOOL_MODE_DEFINEBYQUERYMERGECOPY, + MERGETOOL_MODE_LISTMERGEDVODBCOPY, + MERGETOOL_MODE_REVERTMERGEDVODBCOPY, + MERGETOOL_MODE_UPDATEMERGEDVODBCOPY +} mergetoolMode; + +pxConfig *mergetoolConfig(pxConfig *config, int argc, char **argv); + +#endif // MERGETOOL_H Index: branches/eam_branches/ipp-20111122/ippTools/src/mergetoolConfig.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/mergetoolConfig.c (revision 33638) +++ branches/eam_branches/ipp-20111122/ippTools/src/mergetoolConfig.c (revision 33638) @@ -0,0 +1,196 @@ +/* + * addtoolConfig.c + * + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include + +#include "pxtools.h" +#include "pxmerge.h" +#include "mergetool.h" + +pxConfig *mergetoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + // pxcamSetSearchArgs(definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-mergedvodb", PS_META_DUPLICATE_OK, "search by mergedvodbRun.mergedvodb and use as mergedvodb", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-minidvodb_group", PS_META_DUPLICATE_OK, "search by minidvodbRun.minidvodb_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define workdir", NULL); + // psMetadataAddStr(definebyqueryArge, PS_LIST_TAIL, "-set_mergedvodb", 0, "define mergedvodb"); + + // psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psMetadata *updaterunArgs = psMetadataAlloc(); + + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb", 0); + psMetadataAddU64(updaterunArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "change the state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define workdir", NULL); + + + psMetadata *pendingmergeArgs = psMetadataAlloc(); + psMetadataAddU64(pendingmergeArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + psMetadataAddStr(pendingmergeArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", NULL); + psMetadataAddStr(pendingmergeArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb", NULL); + + psMetadataAddU64(pendingmergeArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(pendingmergeArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + + + psMetadata *addmergedArgs = psMetadataAlloc(); + psMetadataAddU64(addmergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id (required)", 0); + psMetadataAddStr(addmergedArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb (required)", 0); + psMetadataAddF32(addmergedArgs, PS_LIST_TAIL, "-dtime_verify", 0, "define elapsed time for DVO verify (seconds)", NAN); + psMetadataAddF32(addmergedArgs, PS_LIST_TAIL, "-dtime_merge", 0, "define elapsed time for DVO merge (seconds)", NAN); + psMetadataAddF32(addmergedArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time for script (seconds)", NAN); + psMetadataAddTime(addmergedArgs, PS_LIST_TAIL, "-epoch", 0, "time merge is finished", NULL); + psMetadataAddS16(addmergedArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + + + + psMetadata *listmergedArgs = psMetadataAlloc(); + psMetadataAddU64(listmergedArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + psMetadataAddStr(listmergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", NULL); + psMetadataAddStr(listmergedArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by minidvodbRun.minidvodb_group", NULL); + psMetadataAddU64(listmergedArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(listmergedArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + psMetadataAddBool(listmergedArgs, PS_LIST_TAIL, "-faulted", 0, "limit to faulted state", false); + + + psMetadata *revertmergedArgs = psMetadataAlloc(); + psMetadataAddU64(revertmergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddS16(revertmergedArgs, PS_LIST_TAIL, "-fault", 0, "search by minidvodbCopyfault", 0); + psMetadataAddStr(revertmergedArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb", NULL); + + psMetadata *updatemergedArgs = psMetadataAlloc(); + + psMetadataAddU64(updatemergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddS16(updatemergedArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); + psMetadataAddF32(updatemergedArgs, PS_LIST_TAIL, "-set_dtime_verify", 0, "define elapsed time for DVO verify (seconds)", 0); + psMetadataAddF32(updatemergedArgs, PS_LIST_TAIL, "-set_dtime_merge", 0, "define elapsed time for DVO merge (seconds)", 0); + psMetadataAddF32(updatemergedArgs, PS_LIST_TAIL, "-set_dtime_script", 0, "define elapsed time for script (seconds)", 0); + psMetadataAddStr(updatemergedArgs, PS_LIST_TAIL, "-set_state", 0, "change the state", NULL); + // -definebyquerymergecopy + psMetadata *definebyquerymergecopyArgs = psMetadataAlloc(); + psMetadataAddS64(definebyquerymergecopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(definebyquerymergecopyArgs, PS_LIST_TAIL, "-mergedvodb", PS_META_DUPLICATE_OK, "search by mergedvodb", NULL); + psMetadataAddStr(definebyquerymergecopyArgs, PS_LIST_TAIL, "-set_mergedvodb_rsync_path", 0, "define workdir", NULL); + psMetadataAddStr(definebyquerymergecopyArgs, PS_LIST_TAIL, "-set_destination_host", 0, "set destination host", NULL); + psMetadataAddBool(definebyquerymergecopyArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyquerymergecopyArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definebyquerymergecopyArgs, PS_LIST_TAIL, "-last_merged", 0, "if multiple minidvodbs have been dvomerged between rsyncs, then only queue the most recent merged dvodb for copying", false); + + psMetadata *listmergedvodbcopyArgs = psMetadataAlloc(); + psMetadataAddU64(listmergedvodbcopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by mergedvodb_id", 0); + psMetadataAddStr(listmergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodbcopy_id", 0, "search by mergedvodbcopy_id", NULL); + psMetadataAddStr(listmergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodbRun.mergedvodb", NULL); + psMetadataAddStr(listmergedvodbcopyArgs, PS_LIST_TAIL, "-destination_host", 0, "search by mergedvodbCopy.destination_host", NULL); + psMetadataAddBool(listmergedvodbcopyArgs, PS_LIST_TAIL, "-pending", 0, "limit to pending items", false); + psMetadataAddU64(listmergedvodbcopyArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(listmergedvodbcopyArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + psMetadataAddBool(listmergedvodbcopyArgs, PS_LIST_TAIL, "-faulted", 0, "limit to faulted state", false); + + psMetadata *revertmergedvodbcopyArgs = psMetadataAlloc(); + psMetadataAddU64(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodbcopy_id", 0, "search by mergedvodbcopy_id", 0); + psMetadataAddU64(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-destination_host", 0, "search by destination_host", NULL); + psMetadataAddS16(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-fault", 0, "search by mergedvodbCopyfault", 0); + psMetadata *updatemergedvodbcopyArgs = psMetadataAlloc(); + psMetadataAddU64(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodbcopy_id", 0, "search by mergedvodbcopy_id", 0); + psMetadataAddU64(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-host", 0, "search by host", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodb_rsync_path", 0, "search by rsync", NULL); + psMetadataAddS16(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + + + psMetadataAddS16(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); + + psMetadataAddF32(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_dtime", 0, "set elapsed time for transfer", 0); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_mergedvodb_rsync_path", 0, "change the mergedvodb_rsync_path", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_destination_host", 0, "change the host", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_state", 0, "change the state", NULL); + + + + + + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from minindvodbRun", MERGETOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "update existing mergedvodbRuns", MERGETOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingmerge", "list pending mergedvodbRuns", MERGETOOL_MODE_PENDINGMERGE, pendingmergeArgs); + PXOPT_ADD_MODE("-addmerged", "add processed mergedvodbRun", MERGETOOL_MODE_ADDMERGED, addmergedArgs); + PXOPT_ADD_MODE("-listmerged","list merged dvodbs", MERGETOOL_MODE_LISTMERGED, listmergedArgs); + PXOPT_ADD_MODE("-revertmerged","revert merged dvodbs", MERGETOOL_MODE_REVERTMERGED, revertmergedArgs); + PXOPT_ADD_MODE("-updatemerged","change merged dvodb properties",MERGETOOL_MODE_UPDATEMERGED, updatemergedArgs); + PXOPT_ADD_MODE("-definebyquerymergecopy", "create runs from mergedvodbRun", MERGETOOL_MODE_DEFINEBYQUERYMERGECOPY, definebyquerymergecopyArgs); + PXOPT_ADD_MODE("-listmergedvodbcopy","list copy mergedvodbs", MERGETOOL_MODE_LISTMERGEDVODBCOPY, listmergedvodbcopyArgs); + PXOPT_ADD_MODE("-revertmergedvodbcopy","revert copy mergedvobs",MERGETOOL_MODE_REVERTMERGEDVODBCOPY, revertmergedvodbcopyArgs); + PXOPT_ADD_MODE("-updatemergedvodbcopy","change mergedvodb copy properties",MERGETOOL_MODE_UPDATEMERGEDVODBCOPY, updatemergedvodbcopyArgs); + + + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: branches/eam_branches/ipp-20111122/ippTools/src/minidvodbtool.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/minidvodbtool.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/minidvodbtool.c (revision 33638) @@ -89,5 +89,5 @@ psMetadata *where = psMetadataAlloc(); pxcamGetSearchArgs (config, where); - PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodbRun_id", "=="); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodb_id", "=="); pxAddLabelSearchArgs (config, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "=="); // define using camRun label Index: branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.c (revision 33638) @@ -704,6 +704,8 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); - PXOPT_LOOKUP_S64(options, config->args, "-options", false, false); + PXOPT_LOOKUP_S64(options, config->args, "-options", false, false); PXOPT_LOOKUP_S64(dep_id, config->args, "-dep_id", false, false); + PXOPT_LOOKUP_S64(parent_id, config->args, "-parent_id", false, false); + PXOPT_LOOKUP_BOOL(is_parent, config->args, "-is_parent", false); // unless the job is being inserted with stop state require outputBase @@ -716,4 +718,19 @@ if (!strcmp(job_type, "get_image") || !strcmp(job_type, "detect_query") || !strcmp(job_type, "none")) { stampJob = false; + } else if (!strcmp(job_type, "child")) { + // job_type child's only action is to resolve a dependent + // XXX: IS this necessary? + if (!dep_id) { + psError(PS_ERR_UNKNOWN, true, "dep_id required for child job\n"); + return false; + } + if (is_parent) { + psError(PS_ERR_UNKNOWN, true, "job type child can not be a parent job\n"); + return false; + } + stampJob = false; + } else if (!strcmp(job_type, "mosaic")) { + stampJob = false; + is_parent = true; } else if (!strcmp(job_type, "stamp")) { stampJob = true; @@ -723,5 +740,5 @@ } if (!pstampJobInsert(config->dbh, - 0, // job_id + 0, // job_id req_id, rownum, @@ -733,5 +750,7 @@ options, dep_id, - 0 // fault_count + 0, // fault_count + parent_id, + is_parent )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -761,4 +780,5 @@ PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); + PXOPT_COPY_STR(config->args, where, "-jobType", "jobType", "=="); PXOPT_COPY_S64(config->args, where, "-fault", "fault", "=="); @@ -842,8 +862,11 @@ // use psDBGenerateWhereSQL because the SQL yields an intermediate table + psString whereStr = NULL; if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&whereStr, "\n AND %s", whereClause); psFree(whereClause); + } else { + psStringAppend(&whereStr, "%s", ""); } psFree(where); @@ -858,9 +881,11 @@ } - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } + if (!p_psDBRunQueryF(config->dbh, query, whereStr, whereStr)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereStr); + psFree(query); + return false; + } + psFree(whereStr); psFree(query); @@ -956,4 +981,5 @@ PXOPT_LOOKUP_S32(fault, config->args, "-set_fault", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); psMetadata *where = psMetadataAlloc(); @@ -986,5 +1012,5 @@ psFree(where); - if (!p_psDBRunQueryF(config->dbh, query, fault, fault)) { + if (!p_psDBRunQueryF(config->dbh, query, fault, fault, state)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); Index: branches/eam_branches/ipp-20111122/ippTools/src/pstamptoolConfig.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/pstamptoolConfig.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/pstamptoolConfig.c (revision 33638) @@ -144,4 +144,6 @@ psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-options", 0, "define options", 0); psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-dep_id", 0, "define job dep_id", 0); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-parent_id", 0, "define parent's job_id", 0); + psMetadataAddBool(addjobArgs, PS_LIST_TAIL, "-is_parent", 0, "define whether job has children", false); psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0, "define job result", 0); @@ -152,4 +154,5 @@ psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-dep_id", 0, "select by dependent ID", 0); psMetadataAddS16(listjobArgs, PS_LIST_TAIL, "-fault", 0, "select by fault", 0); + psMetadataAddStr(listjobArgs, PS_LIST_TAIL, "-jobType", 0, "select by jobType", 0); psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(listjobArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -186,4 +189,5 @@ psMetadataAddS32(stopdependentjobArgs, PS_LIST_TAIL, "-fault_count", 0, "select by fault_count (>=)", 0); psMetadataAddS16(stopdependentjobArgs, PS_LIST_TAIL, "-set_fault", 0, "new fault value for job and dependent (required)", 0); + psMetadataAddStr(stopdependentjobArgs, PS_LIST_TAIL, "-set_state", 0, "new pstampDependent.state", "new"); psMetadataAddStr(stopdependentjobArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL); psMetadataAddU64(stopdependentjobArgs, PS_LIST_TAIL, "-limit", 0, "not used", 0); Index: branches/eam_branches/ipp-20111122/ippTools/src/pxadd.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/pxadd.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/pxadd.c (revision 33638) @@ -157,8 +157,4 @@ } if (strcmp(stage,"staticsky") == 0) { - query = pxDataGet("addtool_queue_sky_id.sql"); - psMemSetPersistent(query, true); - } - if (strcmp(stage,"staticsky_multi") == 0) { query = pxDataGet("addtool_queue_sky_id_multi.sql"); psMemSetPersistent(query, true); Index: branches/eam_branches/ipp-20111122/ippTools/src/pxchip.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/pxchip.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/pxchip.c (revision 33638) @@ -71,5 +71,5 @@ psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); - psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object (LIKE comparison)", NULL); psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); psMetadataAddStr(md, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL); Index: branches/eam_branches/ipp-20111122/ippTools/src/pxmerge.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/pxmerge.c (revision 33638) +++ branches/eam_branches/ipp-20111122/ippTools/src/pxmerge.c (revision 33638) @@ -0,0 +1,97 @@ +/* + * pxadd.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxmerge.h" + +bool pxmergeQueueByMinidvodbID(pxConfig *config, + psS64 minidvodb_id, + char *mergedvodb, + char *mergedvodb_path + ) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("mergetool_queue_merge_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + } + if (!p_psDBRunQueryF(config->dbh,"INSERT INTO mergedvodbRun SELECT 0, minidvodb_id, '%s', '%s', 'new', NULL FROM minidvodbRun WHERE minidvodbRun.minidvodb_id = %lld and minidvodbRun.state = 'merged'", mergedvodb, mergedvodb_path, (long long) minidvodb_id)) + { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + return false; + } + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row \n%s", query); + return false; + } + return true; +} + +bool pxaddQueueByMergeID(pxConfig *config, + psS64 merge_id, + char *destination_host, + char *mergedvodb_rsync_path + ) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("mergetool_queue_mergecopy_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + } + + if (!p_psDBRunQueryF(config->dbh,"INSERT INTO mergedvodbCopy SELECT 0, merge_id, '%s', '%s', 0, 'new', NULL, 0 FROM mergedvodbRun WHERE mergedvodbRun.merge_id = %lld and (mergedvodbRun.state = 'merged' or mergedvodbRun.state = 'full')", mergedvodb_rsync_path, destination_host, (long long) merge_id)) + + { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row \n%s", query); + return false; + } + + return true; +} Index: branches/eam_branches/ipp-20111122/ippTools/src/pxmerge.h =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/pxmerge.h (revision 33638) +++ branches/eam_branches/ipp-20111122/ippTools/src/pxmerge.h (revision 33638) @@ -0,0 +1,41 @@ +/* + * pxadd.h + * + * Copyright (C) 2009 Joshua Hoblitt, Chris Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXMERGE_H +#define PXMERGE_H 1 + +#include + +#include "pxtools.h" + + +bool pxmergeQueueByMinidvodbID(pxConfig *config, + psS64 minidvodb_id, + char *mergedvodbb, + char *mergedvodb_path + ); + +bool pxaddQueueByMergeID(pxConfig *config, + psS64 merge_id, + char *destination_host, + char *mergedvodb_rsync_path + ); + + +#endif // PXMERGE_H Index: branches/eam_branches/ipp-20111122/ippTools/src/pztool.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/pztool.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/pztool.c (revision 33638) @@ -338,8 +338,8 @@ } - psStringAppend(&query, " ORDER BY dateobs"); - if (desc) { - psStringAppend(&query, " DESC"); - } +/* psStringAppend(&query, " ORDER BY dateobs"); */ +/* if (desc) { */ +/* psStringAppend(&query, " DESC"); */ +/* } */ // request the full "limit" from each known camera and throw away any Index: branches/eam_branches/ipp-20111122/ippTools/src/staticskytool.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/staticskytool.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/staticskytool.c (revision 33638) @@ -38,7 +38,15 @@ static bool resultMode(pxConfig *config); static bool revertMode(pxConfig *config); -static bool updateresult(pxConfig *config); +static bool updateresultMode(pxConfig *config); static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); + +static bool defineskycalrunMode(pxConfig *config); +static bool updateskycalrunMode(pxConfig *config); +static bool pendingskycalrunMode(pxConfig *config); +static bool addskycalresultMode(pxConfig *config); +static bool skycalresultMode(pxConfig *config); +static bool revertskycalresultMode(pxConfig *config); +static bool updateskycalresultMode(pxConfig *config); static bool setstaticskyRunState(pxConfig *config, psS64 sky_id, const char *state); @@ -69,7 +77,14 @@ MODECASE(STATICSKYTOOL_MODE_RESULT, resultMode); MODECASE(STATICSKYTOOL_MODE_REVERT, revertMode); - MODECASE(STATICSKYTOOL_MODE_UPDATERESULT, updateresult); + MODECASE(STATICSKYTOOL_MODE_UPDATERESULT, updateresultMode); MODECASE(STATICSKYTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(STATICSKYTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(STATICSKYTOOL_MODE_DEFINESKYCALRUN, defineskycalrunMode); + MODECASE(STATICSKYTOOL_MODE_UPDATESKYCALRUN, updateskycalrunMode); + MODECASE(STATICSKYTOOL_MODE_PENDINGSKYCALRUN, pendingskycalrunMode); + MODECASE(STATICSKYTOOL_MODE_ADDSKYCALRESULT, addskycalresultMode); + MODECASE(STATICSKYTOOL_MODE_UPDATESKYCALRESULT,updateskycalresultMode); + MODECASE(STATICSKYTOOL_MODE_REVERTSKYCALRESULT,revertskycalresultMode); + MODECASE(STATICSKYTOOL_MODE_SKYCALRESULT, skycalresultMode); default: psAbort("invalid option (this should not happen)"); @@ -97,10 +112,8 @@ { PS_ASSERT_PTR_NON_NULL(config, false); - - // required options - PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); // optional - PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); @@ -111,5 +124,6 @@ psMetadata *whereMD = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_S64(config->args, whereMD, "-select_stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "stackRun.skycell_id", "LIKE"); PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id", "stackRun.tess_id", "=="); PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">="); @@ -621,5 +635,8 @@ PXOPT_COPY_STR(config->args, where, "-label", "staticskyRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-data_group", "staticskyRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "=="); + PXOPT_LOOKUP_S32(num_filters, config->args, "-num_filters", false, false); PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); @@ -645,8 +662,14 @@ psFree(where); + psStringAppend(&query, "\nGROUP BY sky_id"); + if (num_filters) { + psStringAppend(&query, "\nHAVING COUNT(filter) >= %d", num_filters); + } + + // treat limit == 0 as "no limit" if (limit) { psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); + psStringAppend(&query, "\n%s", limitString); psFree(limitString); } @@ -735,5 +758,5 @@ } -static bool updateresult(pxConfig *config) +static bool updateresultMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -978,2 +1001,477 @@ # endif +static bool defineskycalrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + // none required. We get workdir, etc from staticskyRun if not provided + + // optional + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + + psMetadata *whereMD = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, whereMD, "-select_sky_id", "staticskyRun.sky_id", "=="); + PXOPT_COPY_S64(config->args, whereMD, "-select_stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">="); + pxAddLabelSearchArgs(config, whereMD, "-select_label", "stackRun.label", "LIKE"); + pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "stackRun.data_group", "LIKE"); + pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); + + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + psString query = pxDataGet("staticskytool_defineskycalrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(whereMD); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&query, "\nAND %s", whereClause); + psFree(whereClause); + psFree(whereMD); + + if (!rerun) { + if (label) { + psStringAppend(&query, "\nAND (skycalRun.label IS NULL OR skycalRun.label = '%s')", label); + } + psStringAppend(&query, "\nAND skycal_id IS NULL"); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + // we now have a list of (tess_id, skycell_id) that (potentially) meet out needs + // we now need to loop over all of these and for each pair, select the best set of + // inputs + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + return false; + } + if (!psArrayLength(output)) { + psWarning("staticskytool: no rows found"); + psFree(output); + return true; + } + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "skycalRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row from select + bool status; + + psS64 sky_id = psMetadataLookupS64(&status, row, "sky_id"); + psS64 stack_id = psMetadataLookupS64(&status, row, "stack_id"); + psString sky_workdir = psMetadataLookupStr(&status, row, "workdir"); + psString sky_label = psMetadataLookupStr(&status, row, "label"); + psString sky_data_group = psMetadataLookupStr(&status, row, "data_group"); + + // create a staticskyRun + if (!skycalRunInsert(config->dbh, + 0x0, // skycal_id + sky_id, + stack_id, + "new", // state + workdir ? workdir : sky_workdir, + label ? label : sky_label, + data_group ? data_group : sky_data_group, + dist_group, + reduction, + registered, + note + ) + ) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + } + psFree(output); + return true; +} + +static bool updateskycalrunMode(pxConfig *config) +{ +#ifdef notyet + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-sky_id", "sky_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE staticskyRun"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "staticskyRun", "sky_id", "staticskyResult", true, false); + psFree(query); + psFree(where); + + return result; +#endif + return false; +} +static bool pendingskycalrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *whereMD = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, whereMD, "-skycal_id", "skycal_id", "=="); + PXOPT_COPY_S64(config->args, whereMD, "-sky_id", "sky_id", "=="); + pxAddLabelSearchArgs (config, whereMD, "-label", "skycalRun.label", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-filter", "stackRun.filter", "LIKE"); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("staticskytool_pendingskycalrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&query, "\n AND %s", whereClause); + psFree(whereClause); + psFree(whereMD); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n%s", limitString); + psFree(limitString); + } + + // the where clause is required and matches the WHERE hook format string + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("staticskytool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pendingskycalRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return true; +} + +static bool addskycalresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(skycal_id, config->args, "-skycal_id", true, false); + + // optional + PXOPT_LOOKUP_F32(sigma_ra, config->args, "-sigma_ra", false, false); + PXOPT_LOOKUP_F32(sigma_dec, config->args, "-sigma_dec", false, false); + + PXOPT_LOOKUP_F32(zpt_obs, config->args, "-zpt_obs", false, false); + PXOPT_LOOKUP_F32(zpt_stdev, config->args, "-zpt_stdev", false, false); + + + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_F32(dtime_astrom, config->args, "-dtime_astrom", false, false); + + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); +// PXOPT_LOOKUP_S32(n_stars, config->args, "-n_stars", false, false); + PXOPT_LOOKUP_S32(n_astrom, config->args, "-n_astrom", false, false); + + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_psphot, config->args, "-ver_psphot", false, false); + PXOPT_LOOKUP_STR(ver_psastro, config->args, "-ver_psastro", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + + psString software_ver = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + software_ver = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + software_ver = pxMergeCodeVersions(software_ver,ver_psphot); + } + if (ver_psastro) { + software_ver = pxMergeCodeVersions(software_ver,ver_psastro); + } + if (ver_ppstats) { + software_ver = pxMergeCodeVersions(software_ver,ver_ppstats); + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + skycalResultRow *row = skycalResultRowAlloc( + skycal_id, + path_base, + dtime_script, + dtime_astrom, + sigma_ra, + sigma_dec, + n_astrom, + zpt_obs, + zpt_stdev, + quality, + software_ver, + hostname, + fault + ); + + if (!skycalResultInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + return false; + } + + psFree(row); + + if (fault) { + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } + + psString query = "UPDATE skycalRun SET state = 'full' WHERE skycal_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, skycal_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} +static bool updateskycalresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycal_id", "=="); + + if (!pxSetFaultCode(config->dbh, "skycalResult", where, fault, quality)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + return true; +} +static bool skycalresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-sky_id", "skycalRun.sky_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "skycalRun.stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-label", "skycalRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "skycalRun.data_group", "LIKE"); + PXOPT_COPY_S16(config->args, where, "-fault", "skycalResult.fault", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("staticskytool_skycalresult.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!psListLength(where->list)) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required"); + return false; + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("staticskytool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "staticskyResult", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} +static bool revertskycalresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "staticskyResult.sky_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "staticskyRun.label", "=="); + pxAddLabelSearchArgs(config, where, "-data_group", "staticskyRun.data_group", "=="); + pxAddLabelSearchArgs(config, where, "-filter", "stackRun.filter", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // Delete product + psString delete = pxDataGet("staticskytool_revertskycal.sql"); + if (!delete) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + return false; + } + psFree(delete); + + int numRows = psDBAffectedRows(config->dbh); // Number of row affected + psLogMsg("staticskytool", PS_LOG_INFO, "Deleted %d rows", numRows); + + psFree(where); + return true; +} Index: branches/eam_branches/ipp-20111122/ippTools/src/staticskytool.h =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/staticskytool.h (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/staticskytool.h (revision 33638) @@ -34,5 +34,13 @@ STATICSKYTOOL_MODE_UPDATERESULT, STATICSKYTOOL_MODE_EXPORTRUN, - STATICSKYTOOL_MODE_IMPORTRUN + STATICSKYTOOL_MODE_IMPORTRUN, + + STATICSKYTOOL_MODE_DEFINESKYCALRUN, + STATICSKYTOOL_MODE_PENDINGSKYCALRUN, + STATICSKYTOOL_MODE_UPDATESKYCALRUN, + STATICSKYTOOL_MODE_ADDSKYCALRESULT, + STATICSKYTOOL_MODE_UPDATESKYCALRESULT, + STATICSKYTOOL_MODE_SKYCALRESULT, + STATICSKYTOOL_MODE_REVERTSKYCALRESULT, } staticskytoolMode; Index: branches/eam_branches/ipp-20111122/ippTools/src/staticskytoolConfig.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/staticskytoolConfig.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/staticskytoolConfig.c (revision 33638) @@ -48,4 +48,5 @@ // XXX need a 'ra_min,max', 'dec_min,max' : to do this, we need a table of skycell boundaries psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-select_stack_id", 0, "search for stack_id", 0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); @@ -107,8 +108,9 @@ psMetadata *resultArgs= psMetadataAlloc(); psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); - // psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", 0); - // psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs (>=)", 0); psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddU64(resultArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -138,5 +140,102 @@ psMetadata *importrunArgs = psMetadataAlloc(); psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); - + + // -defineskycalrun + psMetadata *defineskycalrunArgs = psMetadataAlloc(); + psMetadataAddS64(defineskycalrunArgs, PS_LIST_TAIL, "-select_sky_id", 0, "search for sky_id", 0); + psMetadataAddS64(defineskycalrunArgs, PS_LIST_TAIL, "-select_stack_id", 0, "search for stack_id", 0); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); + psMetadataAddF32(defineskycalrunArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddTime(defineskycalrunArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-rerun", 0, "queue new run even if one already exists for inputs", false); + psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-check_inputs", 0, "list inputs, do not modify database", false); + psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updateskycalrun + psMetadata *updateskycalrunArgs = psMetadataAlloc(); + psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-sky_id", 0, "search by stack ID", 0); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + + // -pendingskycalrun + psMetadata *pendingskycalrunArgs = psMetadataAlloc(); + psMetadataAddS64(pendingskycalrunArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0); + psMetadataAddS64(pendingskycalrunArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); + psMetadataAddStr(pendingskycalrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddStr(pendingskycalrunArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddU64(pendingskycalrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingskycalrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + + // -addskycalresult + psMetadata *addskycalresultArgs = psMetadataAlloc(); + psMetadataAddS64(addskycalresultArgs, PS_LIST_TAIL, "-skycal_id", 0, "define camtool ID (required)", 0); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-sigma_ra", 0, "define exposure E ra", NAN); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-sigma_dec", 0, "define exposure E dec", NAN); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-zpt_obs", 0, "define observed zero point", 0); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-zpt_stdev", 0, "define observed zero point stdandard deviation", 0); + + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-dtime_astrom", 0, "define elapsed time for astrometry (seconds)", NAN); + + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL); + // psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_stars", 0, "define number of stars", 0); + psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_astrom", 0, "define number of astrometry reference objects", 0); + + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddS16(addskycalresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + psMetadataAddS16(addskycalresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL); + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL); + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_psphot", 0, "define psphot version", NULL); + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_psastro", 0, "define psastro version", NULL); + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL); + + // -revertskycal + psMetadata *revertskycalArgs= psMetadataAlloc(); + psMetadataAddS64(revertskycalArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by staticsky ID", 0); + psMetadataAddStr(revertskycalArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddStr(revertskycalArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by data_group", 0); + psMetadataAddStr(revertskycalArgs, PS_LIST_TAIL, "-filter", PS_META_DUPLICATE_OK, "search by filter", 0); + psMetadataAddS16(revertskycalArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertskycalArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", false); + + // -updateskycalresult + psMetadata *updateskycalresultArgs = psMetadataAlloc(); + psMetadataAddS64(updateskycalresultArgs, PS_LIST_TAIL, "-skycal_id", 0, "define staticksky ID (required)", 0); + psMetadataAddS16(updateskycalresultArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code (required)", 0); + psMetadataAddS16(updateskycalresultArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality code", 0); + + // -skycalresult + psMetadata *skycalresultArgs= psMetadataAlloc(); + psMetadataAddS64(skycalresultArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0); + psMetadataAddS64(skycalresultArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); + psMetadataAddS64(skycalresultArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); + psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); + psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", 0); + psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-label", 0, "search by label (LIKE comparison)", NULL); + psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddS16(skycalresultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddU64(skycalresultArgs, PS_LIST_TAIL, "-limit", 0, "limit skycalresult set to N items", 0); + psMetadataAddBool(skycalresultArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psFree(now); @@ -155,4 +254,11 @@ PXOPT_ADD_MODE("-exportrun", "Export run", STATICSKYTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "Import run", STATICSKYTOOL_MODE_IMPORTRUN, importrunArgs); + PXOPT_ADD_MODE("-defineskycalrun", "Define a new skycalrun", STATICSKYTOOL_MODE_DEFINESKYCALRUN, defineskycalrunArgs); + PXOPT_ADD_MODE("-updateskycalrun", "Update a skycalrun", STATICSKYTOOL_MODE_UPDATESKYCALRUN, updateskycalrunArgs); + PXOPT_ADD_MODE("-pendingskycalrun", "Get skcal runs to do", STATICSKYTOOL_MODE_PENDINGSKYCALRUN, pendingskycalrunArgs); + PXOPT_ADD_MODE("-addskycalresult", "add skycal result", STATICSKYTOOL_MODE_ADDSKYCALRESULT, addskycalresultArgs); + PXOPT_ADD_MODE("-revertskycal", "revert faulted skycal run", STATICSKYTOOL_MODE_REVERTSKYCALRESULT, revertskycalArgs); + PXOPT_ADD_MODE("-updateskycal", "revert faulted skycal run", STATICSKYTOOL_MODE_UPDATESKYCALRESULT, updateskycalresultArgs); + PXOPT_ADD_MODE("-skycalresult", "Get result of skycal run", STATICSKYTOOL_MODE_SKYCALRESULT, skycalresultArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/eam_branches/ipp-20111122/ippTools/src/warptool.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/warptool.c (revision 32851) +++ branches/eam_branches/ipp-20111122/ippTools/src/warptool.c (revision 33638) @@ -2581,4 +2581,13 @@ psFree(query); - return true; -} + if (!skycell_id) { + // If we are updateing a whole warpRun set skycells with bad quality to 'full' + query = "UPDATE warpSkyfile SET data_state ='full', fault = 0 WHERE warp_id = %" PRId64 " AND quality != 0 AND (data_state != 'full')"; + if (!p_psDBRunQueryF(config->dbh, query, warp_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + return true; +}