Index: trunk/ippTools/share/Makefile.am =================================================================== --- trunk/ippTools/share/Makefile.am (revision 33024) +++ trunk/ippTools/share/Makefile.am (revision 33030) @@ -310,4 +310,14 @@ minidvodbtool_queue_minidvodb_id.sql \ minidvodbtool_revertminidvodbcopy.sql \ + mergetool_find_minidvodb_id.sql \ + mergetool_find_mergedvodbcopy.sql \ + mergetool_find_merge_id_dvo.sql \ + mergetool_find_pendingmergeprocess.sql \ + mergetool_find_pending_merge.sql \ + mergetool_find_processed.sql \ + mergetool_queue_mergecopy_id.sql \ + mergetool_queue_merge_id.sql \ + mergetool_revertmergedvodbcopy.sql \ + mergetool_revertmergedvodbprocessed.sql \ pstamptool_completedreq.sql \ pstamptool_datastore.sql \ Index: trunk/ippTools/share/addtool_checkminidvodbaddrun.sql =================================================================== --- trunk/ippTools/share/addtool_checkminidvodbaddrun.sql (revision 33024) +++ trunk/ippTools/share/addtool_checkminidvodbaddrun.sql (revision 33030) @@ -1,4 +1,4 @@ -- this selects the minidvodbs that have complete addRuns -SELECT minidvodb_id, minidvodb_name, minidvodb_path, mergedvodb_path, state, minidbg +SELECT minidvodb_id, minidvodb_name, minidvodb_path, state, minidbg AS minidvodb_group, (cnt - cnt2) as not_full, cnt as addRun_count FROM minidvodbRun Index: trunk/ippTools/share/addtool_find_minidvodbrun2.sql =================================================================== --- trunk/ippTools/share/addtool_find_minidvodbrun2.sql (revision 33024) +++ trunk/ippTools/share/addtool_find_minidvodbrun2.sql (revision 33030) @@ -1,4 +1,4 @@ -- this selects the minidvodbs that have complete addRuns -SELECT minidvodb_name, minidvodb_path, mergedvodb_path, state, minidbg +SELECT minidvodb_name, minidvodb_path, state, minidbg AS minidvodb_group FROM minidvodbRun Index: trunk/ippTools/share/addtool_queue_minidvodbrun.sql =================================================================== --- trunk/ippTools/share/addtool_queue_minidvodbrun.sql (revision 33024) +++ trunk/ippTools/share/addtool_queue_minidvodbrun.sql (revision 33030) @@ -4,5 +4,4 @@ minidvodb_name, minidvodb_path, - mergedvodb_path, 'new' Index: trunk/ippTools/share/mergetool_find_merge_id_dvo.sql =================================================================== --- trunk/ippTools/share/mergetool_find_merge_id_dvo.sql (revision 33030) +++ trunk/ippTools/share/mergetool_find_merge_id_dvo.sql (revision 33030) @@ -0,0 +1,9 @@ +SELECT mergedvodbRun.* FROM mergedvodbRun +JOIN mergedvodbProcessed USING(merge_id) +WHERE ( mergedvodbRun.state = 'merged' OR mergedvodbRun.state = 'full') + AND mergedvodbProcessed.fault = 0 + AND merge_id NOT IN (SELECT merge_id + FROM mergedvodbCopy + JOIN mergedvodbRun USING(mergedvodb_id) + WHERE %s + ) Index: trunk/ippTools/share/mergetool_find_mergedvodbcopy.sql =================================================================== --- trunk/ippTools/share/mergetool_find_mergedvodbcopy.sql (revision 33030) +++ trunk/ippTools/share/mergetool_find_mergedvodbcopy.sql (revision 33030) @@ -0,0 +1,7 @@ +SELECT + mergedvodbCopy.*, + mergedvodbRun.mergedvodb_path +FROM mergedvodbCopy + JOIN mergedvodbRun using (mergedvodb_id) + + Index: trunk/ippTools/share/mergetool_find_minidvodb_id.sql =================================================================== --- trunk/ippTools/share/mergetool_find_minidvodb_id.sql (revision 33030) +++ trunk/ippTools/share/mergetool_find_minidvodb_id.sql (revision 33030) @@ -0,0 +1,8 @@ +SELECT minidvodbRun.* FROM minidvodbRun +JOIN minidvodbProcessed USING(minidvodb_id) +WHERE minidvodbRun.state = 'merged' and minidvodbProcessed.fault = 0 + AND minidvodb_id NOT IN (SELECT minidvodb_id + FROM mergedvodbRun + JOIN minidvodbRun USING(minidvodb_id) + WHERE %s + ) Index: trunk/ippTools/share/mergetool_find_pending_merge.sql =================================================================== --- trunk/ippTools/share/mergetool_find_pending_merge.sql (revision 33030) +++ trunk/ippTools/share/mergetool_find_pending_merge.sql (revision 33030) @@ -0,0 +1,14 @@ +SELECT + mergedvodbRun.*, + minidvodbRun.minidvodb_path +FROM minidvodbRun +JOIN mergedvodbRun + USING(minidvodb_id) +LEFT JOIN mergedvodbProcessed + USING(merge_id) +WHERE + (mergedvodbRun.state = 'new' AND mergedvodbProcessed.merge_id IS NULL) + AND mergedvodbRun.mergedvodb IS NOT NULL + AND mergedvodbRun.mergedvodb_path IS NOT NULL + + Index: trunk/ippTools/share/mergetool_find_pendingmergeprocess.sql =================================================================== --- trunk/ippTools/share/mergetool_find_pendingmergeprocess.sql (revision 33030) +++ trunk/ippTools/share/mergetool_find_pendingmergeprocess.sql (revision 33030) @@ -0,0 +1,5 @@ +SELECT mergedvodbRun.* +FROM mergedvodbRun +LEFT JOIN mergedvodbProcessed +USING(merge_id) +WHERE (mergedvodbRun.state = 'new' AND mergedvodbProcessed.merge_id IS NULL) Index: trunk/ippTools/share/mergetool_find_processed.sql =================================================================== --- trunk/ippTools/share/mergetool_find_processed.sql (revision 33030) +++ trunk/ippTools/share/mergetool_find_processed.sql (revision 33030) @@ -0,0 +1,7 @@ +SELECT + mergedvodbProcessed.*, + mergedvodbRun.mergedvodb_path +FROM mergedvodbProcessed +JOIN mergedvodbRun + USING(merge_id) + Index: trunk/ippTools/share/mergetool_queue_merge_id.sql =================================================================== --- trunk/ippTools/share/mergetool_queue_merge_id.sql (revision 33030) +++ trunk/ippTools/share/mergetool_queue_merge_id.sql (revision 33030) @@ -0,0 +1,11 @@ +INSERT INTO mergedvodbRun + SELECT + 0, -- merge_id + minidvodb_id, -- minidvodb_id + '%s', -- mergedvodb + '%s', -- mergedvodb_path + '%s', -- state + FROM minidvodbRun join minidvodbProcessed using (minidvodb_id) + WHERE + minidvodbRun.state = 'merged' and minidvodbProcessed.fault = 0 + AND minidvodbRun.minidvodb_id = %lld Index: trunk/ippTools/share/mergetool_queue_mergecopy_id.sql =================================================================== --- trunk/ippTools/share/mergetool_queue_mergecopy_id.sql (revision 33030) +++ trunk/ippTools/share/mergetool_queue_mergecopy_id.sql (revision 33030) @@ -0,0 +1,16 @@ +INSERT INTO mergedvodbCopy + SELECT + 0, -- mergedvodbcopy_id + merge_id, -- merge_id + '%s', -- mergedvodb_rsync_path + '%s', -- destination_host + %d, -- fault + '%s', -- state + %s, -- epoch + %f -- dtime + + FROM mergedvodbRun + WHERE + (mergedvodbRun.state = 'merged' + OR mergedvodbRun.state = 'full') + AND mergedvodbRun.merge_id = %lld Index: trunk/ippTools/share/mergetool_queue_minidvodb_id.sql =================================================================== --- trunk/ippTools/share/mergetool_queue_minidvodb_id.sql (revision 33030) +++ trunk/ippTools/share/mergetool_queue_minidvodb_id.sql (revision 33030) @@ -0,0 +1,15 @@ +INSERT INTO minidvodbCopy + SELECT + 0, -- minidvodbcopy_id + minidvodb_id, -- minidvodb_id + '%s', -- minidvodb_rsync_path + '%s', -- destination_host + %d, -- fault + '%s', -- state + %s, -- epoch + %f -- dtime + + FROM minidvodbRun + WHERE + minidvodbRun.state = 'merged' + AND minidvodbRun.minidvodb_id = %lld Index: trunk/ippTools/share/mergetool_revertmergedvodbcopy.sql =================================================================== --- trunk/ippTools/share/mergetool_revertmergedvodbcopy.sql (revision 33030) +++ trunk/ippTools/share/mergetool_revertmergedvodbcopy.sql (revision 33030) @@ -0,0 +1,7 @@ +UPDATE mergedvodbCopy + -- SET minidvodbCopy.state = 'new', + SET mergedvodbCopy.fault = 0 +WHERE + mergedvodbCopy.state = 'new' + AND mergedvodbCopy.fault >0 + Index: trunk/ippTools/share/mergetool_revertmergedvodbprocessed.sql =================================================================== --- trunk/ippTools/share/mergetool_revertmergedvodbprocessed.sql (revision 33030) +++ trunk/ippTools/share/mergetool_revertmergedvodbprocessed.sql (revision 33030) @@ -0,0 +1,6 @@ +DELETE FROM mergedvodbProcessed +USING mergedvodbProcessed, mergedvodbRun +WHERE + mergedvodbProcessed.merge_id = mergedvodbRun.merge_id + AND mergedvodbProcessed.fault != 0 + AND mergedvodbRun.state = 'new' Index: trunk/ippTools/share/minidvodbtool_find_minidvodb_id_dvo.sql =================================================================== --- trunk/ippTools/share/minidvodbtool_find_minidvodb_id_dvo.sql (revision 33024) +++ trunk/ippTools/share/minidvodbtool_find_minidvodb_id_dvo.sql (revision 33030) @@ -1,5 +1,6 @@ SELECT minidvodbRun.* FROM minidvodbRun JOIN minidvodbProcessed USING(minidvodb_id) -WHERE minidvodbRun.state = 'merged' +WHERE minidvodbRun.state = 'merged' + AND minidvodbProcessed.fault = 0 AND minidvodb_id NOT IN (SELECT minidvodb_id FROM minidvodbCopy Index: trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- trunk/ippTools/share/pxadmin_create_tables.sql (revision 33024) +++ trunk/ippTools/share/pxadmin_create_tables.sql (revision 33030) @@ -1964,5 +1964,4 @@ minidvodb_group VARCHAR(64) NOT NULL, minidvodb_path VARCHAR(255) NOT NULL, - mergedvodb_path VARCHAR(255) NOT NULL, state VARCHAR(64) NOT NULL, creation_date TIMESTAMP, @@ -1972,12 +1971,8 @@ CREATE TABLE minidvodbProcessed ( minidvodb_id BIGINT, - merge_order BIGINT, dtime_resort FLOAT, dtime_relphot FLOAT, - dtime_verify FLOAT, - dtime_merge FLOAT, dtime_script FLOAT, epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - mergedvodb_path VARCHAR(255) NOT NULL, fault SMALLINT NOT NULL, UNIQUE KEY (minidvodb_id), @@ -2000,4 +1995,41 @@ FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun(minidvodb_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE mergedvodbRun ( + merge_id BIGINT AUTO_INCREMENT, + minidvodb_id BIGINT, + mergedvodb VARCHAR(64) NOT NULL, + mergedvodb_path VARCHAR(255) NOT NULL, + state VARCHAR(64) NOT NULL, + creation_date TIMESTAMP, + PRIMARY KEY(merge_id), KEY(minidvodb_id), KEY(mergedvodb), INDEX(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE mergedvodbProcessed ( + merge_id BIGINT, + dtime_verify FLOAT, + dtime_merge FLOAT, + dtime_script FLOAT, + epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + fault SMALLINT NOT NULL, + UNIQUE KEY (merge_id), + KEY(merge_id), + FOREIGN KEY(merge_id) REFERENCES mergedvodbRun(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE mergedvodbCopy ( + mergedvodbcopy_id BIGINT AUTO_INCREMENT, + merge_id BIGINT, + mergedvodb_rsync_path VARCHAR(255) NOT NULL, + destination_host VARCHAR(255) NOT NULL, + fault SMALLINT NOT NULL, + state VARCHAR(64) NOT NULL, + epoch TIMESTAMP, + dtime FLOAT, + PRIMARY KEY(mergedvodbcopy_id), + KEY(merge_id), + FOREIGN KEY(merge_id) REFERENCES mergedvodbRun(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE vpRun ( vp_id BIGINT AUTO_INCREMENT, Index: trunk/ippTools/src/addtool.c =================================================================== --- trunk/ippTools/src/addtool.c (revision 33024) +++ trunk/ippTools/src/addtool.c (revision 33030) @@ -292,7 +292,8 @@ } else { //This picks only the unmagicked/uncensored ones - if (strcmp(stage,"cam") == 0) { - psStringAppend(&query, " AND (camRun.magicked = 0)"); - } + //if (strcmp(stage,"cam") == 0) { + //we can now properly handle the magicked case in ippScripts, so we queue camRuns in any magic state now if -uncensored. + //psStringAppend(&query, " AND (camRun.magicked = 0)"); + //} if (strcmp(stage,"stack") == 0) { psStringAppend(&query, " AND (stackRun.magicked = 0)"); @@ -824,5 +825,5 @@ PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-minidvodb_name", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - + PXOPT_LOOKUP_S64(stage_extra1, config->args, "-stage_extra1", false, false); // generate restrictions psMetadata *where = psMetadataAlloc(); @@ -902,5 +903,19 @@ } } - + //if there is a stage_extra1, set it in addRun (it's not known until it is processed) + if (stage_extra1) { + psString setExtra = NULL; + psStringAppend (&setExtra, "UPDATE addRun set stage_extra1 = %" PRId64, stage_extra1); + psStringAppend (&setExtra, " where add_id = %" PRId64, row->add_id); + + if (!p_psDBRunQuery(config->dbh, setExtra)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + + return false; + } + } @@ -1049,5 +1064,5 @@ PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); PXOPT_COPY_S64(config->args, where, "-stage_id", "addRun.stage_id", "=="); - PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); pxcamGetSearchArgs (config, where); pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); @@ -1221,7 +1236,5 @@ PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", true, false); - PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", true, false); - - //optional + //optional PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-set_minidvodb_name", false, false); PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false); @@ -1238,4 +1251,7 @@ if (minidvodb_path) { minidvodbpath = minidvodb_path; + } else { + psError(PS_ERR_UNKNOWN, false, "require minidvodb_path"); + return false; } @@ -1245,6 +1261,5 @@ minidvodb_group, minidvodbpath, - mergedvodb_path, - "new", + "new", 0 )) { @@ -1268,9 +1283,4 @@ psStringAppend(&minidvodb_name, "%s.%" PRIu64,minidvodb_group,minidvodb_id); } - - if (!minidvodb_path) { - psStringAppend(&minidvodb_path,"%s/%s",mergedvodb_path,minidvodb_name); - } - if (minidvodb_path) { psStringAppend(&minidvodb_path,"/%s",minidvodb_name); @@ -1304,5 +1314,4 @@ PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); PXOPT_COPY_STR(config->args, where, "-minidvodb_path", "minidvodb_path", "=="); - PXOPT_COPY_STR(config->args, where, "-mergedvodb_path", "mergedvodb_path", "=="); PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodb_group", "=="); @@ -1310,5 +1319,4 @@ PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", false, false); PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", false, false); @@ -1345,12 +1353,4 @@ } - if (mergedvodb_path) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - psStringAppend(&query, " mergedvodb_path = '%s'", mergedvodb_path); - cnt++; - } - if (minidvodb_group) { if (cnt) { @@ -1396,5 +1396,5 @@ } - psString firstquery = psStringCopy("SELECT * from minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL"); + psString firstquery = psStringCopy("SELECT * from minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL"); psString firstwhereClause = psDBGenerateWhereConditionSQL(where, NULL); @@ -1444,5 +1444,5 @@ //now flip new -> active - psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL "); + psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL "); psStringAppend(&query2, " AND minidvodb_group = '%s' limit 1;", minidvodb_group); if (!p_psDBRunQuery(config->dbh, query2)) { @@ -1621,13 +1621,9 @@ // required PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-minidvodb_id", true, false); - PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-mergedvodb_path", true, false); PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-minidvodb_group", true, false); // optional - PXOPT_LOOKUP_U64(merge_order, config->args, "-merge_order", false, false); PXOPT_LOOKUP_F32(dtime_relphot, config->args, "-dtime_relphot", false, false); PXOPT_LOOKUP_F32(dtime_resort, config->args, "-dtime_resort", false, false); - PXOPT_LOOKUP_F32(dtime_merge, config->args, "-dtime_merge", false, false); - PXOPT_LOOKUP_F32(dtime_verify, config->args, "-dtime_verify", false, false); PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); @@ -1680,12 +1676,8 @@ minidvodbProcessedRow *row = minidvodbProcessedRowAlloc( pendingRow->minidvodb_id, - merge_order, dtime_resort, dtime_relphot, - dtime_verify, - dtime_merge, dtime_script, epoch, - mergedvodb_path, fault ); @@ -1702,67 +1694,4 @@ } - - - - //this finds the # of merged things (for the merge order) - psString query3 = NULL; - psStringAppend(&query3, "select count(*) from minidvodbRun join minidvodbProcessed using (minidvodb_id) where state = 'merged' and minidvodb_group = '%s';", minidvodb_group); - - if (!p_psDBRunQuery(config->dbh, query3)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query3); - return false; - } - psArray *output2 = p_psDBFetchResult(config->dbh); - if (!output2) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output2)) { - psTrace("addtool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - bool status; - psS64 m_order = psMetadataLookupS64(&status, output2->data[0], "count(*)"); - if (!status) { - - psAbort("failed to lookup value for count column"); - return false; - } - psString final = NULL; - psStringAppend(&final, "%" PRIu64, m_order); - //return false; - psFree(query3); - psFree(output2); - - - - //update the merge_order - - psString query4 = NULL; - psStringAppend(&query4, "update minidvodbProcessed set merge_order = %"PRIu64,m_order); - psStringAppend(&query4," where minidvodb_id = %" PRIu64, minidvodb_id); - //printf("%s", query4); - if (!p_psDBRunQuery(config->dbh, query4)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query4); - return false; - } - // -psFree(query4); - - - -// since there is only one exp per 'new' set mindvodbRun.state = 'merged' - psString query2 = NULL ; psStringAppend(&query2, "UPDATE minidvodbRun SET state = 'merged' WHERE minidvodb_id = %'" PRIu64, row->minidvodb_id); @@ -1777,12 +1706,4 @@ return false; } - - - - - - - - psFree(row); psFree(pendingRow); @@ -1797,10 +1718,4 @@ return false; } - - - //print the merge_order (why not!) - printf("%s", final); - psFree(final); - return true; } @@ -1961,10 +1876,7 @@ PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-minidvodb_id", true, false); - PXOPT_LOOKUP_U64(merge_order, config->args, "-set_merge_order", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); PXOPT_LOOKUP_F32(dtime_relphot, config->args, "-set_dtime_relphot", false, false); PXOPT_LOOKUP_F32(dtime_resort, config->args, "-set_dtime_resort", false, false); - PXOPT_LOOKUP_F32(dtime_merge, config->args, "-set_dtime_merge", false, false); -PXOPT_LOOKUP_F32(dtime_verify, config->args, "-set_dtime_verify", false, false); PXOPT_LOOKUP_F32(dtime_script, config->args, "-set_dtime_script", false, false); PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbProcessed.minidvodb_id", "=="); @@ -1986,13 +1898,4 @@ } - if (merge_order) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - - psStringAppend(&query, " merge_order = %" PRId64, merge_order); - cnt++; - } - if (dtime_relphot) { if (cnt) { @@ -2011,19 +1914,5 @@ } - if (dtime_merge) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - psStringAppend(&query, " dtime_merge = %f", dtime_merge); - cnt++; - } -if (dtime_verify) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - psStringAppend(&query, " dtime_verify = %f", dtime_verify); - cnt++; - } -if (dtime_script) { + if (dtime_script) { if (cnt) { psStringAppend(&query, "%s", comma); Index: trunk/ippTools/src/addtoolConfig.c =================================================================== --- trunk/ippTools/src/addtoolConfig.c (revision 33024) +++ trunk/ippTools/src/addtoolConfig.c (revision 33030) @@ -115,4 +115,6 @@ psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN); psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); +psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-stage_extra1", 0, "set stage_extra1", 0); + // -processedexp @@ -165,5 +167,4 @@ psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group", 0, "define minidvodb_group (required)", NULL); psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path", 0, "define path for minidvodb", NULL); - psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define path for the merged dvodb (required)", NULL); psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); @@ -174,9 +175,7 @@ psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "search by minidvodb_name (LIKE)", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_path", 0, "search by path for minidvodb", NULL); - psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-mergedvodb_path", 0, "search by path for the merged dvodb", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_name", 0, "define minidvodb_name", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path", 0, "define path for minidvodb", NULL); - psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define path for minidvodb", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group", 0, "define path for the merged dvodb", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); @@ -205,12 +204,8 @@ psMetadata *addminidvodbprocessedArgs = psMetadataAlloc(); psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "define minidvodb_id (required)", 0); - psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-merge_order", 0, "define merge order", 0); psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_relphot", 0, "define elapsed time for relphot (seconds)", NAN); psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_resort", 0, "define elapsed time for resort (seconds)", NAN); - psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_verify", 0, "define elapsed time for DVO verify (seconds)", NAN); - psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_merge", 0, "define elapsed time for DVO merge (seconds)", NAN); psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time for script (seconds)", NAN); psMetadataAddTime(addminidvodbprocessedArgs, PS_LIST_TAIL, "-epoch", 0, "time merge is finished", NULL); - psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-mergedvodb_path",0, "path of merged dvodb", NULL); psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_group",0, "minidvodb_group", NULL); psMetadataAddS16(addminidvodbprocessedArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); @@ -232,9 +227,6 @@ psMetadataAddStr(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "search by minidvodb_name", NULL); psMetadataAddS16(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); - psMetadataAddU64(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_merge_order", 0, "define merge order", 0); psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_relphot", 0, "define elapsed time for relphot (seconds)", 0); psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_resort", 0, "define elapsed time for resort (seconds)", 0); - psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_verify", 0, "define elapsed time for DVO verify (seconds)", 0); - psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_merge", 0, "define elapsed time for DVO merge (seconds)", 0); psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_script", 0, "define elapsed time for script (seconds)", 0); psMetadata *argSets = psMetadataAlloc(); Index: trunk/ippTools/src/mergetool.c =================================================================== --- trunk/ippTools/src/mergetool.c (revision 33030) +++ trunk/ippTools/src/mergetool.c (revision 33030) @@ -0,0 +1,1227 @@ +/* + * addtool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxmerge.h" +#include "mergetool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool pendingmergeMode(pxConfig *config); +static bool addmergedMode(pxConfig *config); +static bool listmergedMode(pxConfig *config); +static bool revertmergedMode(pxConfig *config); +static bool updatemergedMode(pxConfig *config); +static bool definebyquerymergecopyMode(pxConfig *config); +static bool listmergedvodbcopyMode(pxConfig *config); +static bool revertmergedvodbcopyMode(pxConfig *config); +static bool updatemergedvodbcopyMode(pxConfig *config); + + + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = mergetoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(MERGETOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(MERGETOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(MERGETOOL_MODE_PENDINGMERGE, pendingmergeMode); + MODECASE(MERGETOOL_MODE_ADDMERGED, addmergedMode); + MODECASE(MERGETOOL_MODE_LISTMERGED, listmergedMode); + MODECASE(MERGETOOL_MODE_REVERTMERGED, revertmergedMode); + MODECASE(MERGETOOL_MODE_UPDATEMERGED, updatemergedMode); + MODECASE(MERGETOOL_MODE_DEFINEBYQUERYMERGECOPY, definebyquerymergecopyMode); + MODECASE(MERGETOOL_MODE_LISTMERGEDVODBCOPY, listmergedvodbcopyMode); + MODECASE(MERGETOOL_MODE_REVERTMERGEDVODBCOPY, revertmergedvodbcopyMode); + MODECASE(MERGETOOL_MODE_UPDATEMERGEDVODBCOPY, updatemergedvodbcopyMode); + + + + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodb_id", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "=="); + // PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", false, false); + PXOPT_LOOKUP_STR(mergedvodb, config->args, "-mergedvodb", true, false); + + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + //find the items to queue (uniq minidvodb_id) + psString dvodb_string = NULL; + psString bare_query = NULL; + +bare_query = pxDataGet("mergetool_find_minidvodb_id.sql"); + psStringAppend(&dvodb_string, "mergedvodbRun.mergedvodb = '%s'", mergedvodb); + +if (!bare_query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + psFree(where); + return false; + } + // Take the bare query and add the dvodb selector + psString query = NULL; + psStringAppend(&query, bare_query, dvodb_string); + psFree(dvodb_string); + psFree(bare_query); + + // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; + } + psFree(where); + + + psStringAppend(&query, " GROUP BY minidvodb_id"); + + + psTrace("mergetool.c", PS_LOG_INFO,"query: \n\n%s\n\n",query); + +if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error, \n%s\n", query); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + +for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + minidvodbRunRow *row = minidvodbRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into minidvodbRun"); + psFree(output); + return false; + } + + if (!mergedvodb) { + psError(PS_ERR_UNKNOWN, false, "cannot queue merge run without a defined mergedvodb: minidvodb_id %" PRId64,row->minidvodb_id); + psFree(output); + return false; + } + if (!mergedvodb_path) { + psError(PS_ERR_UNKNOWN, false, "cannot queue merge run without a defined mergedvodb_path: minidvodb_id %" PRId64,row->minidvodb_id); + psFree(output); + return false; + } + + psFree(row); + +} + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } +for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 minidvodb_id =0; + + minidvodbRunRow *row = minidvodbRunObjectFromMetadata(md); + minidvodb_id = row->minidvodb_id; + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into minidvodbRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxmergeQueueByMinidvodbID(config, + minidvodb_id, + mergedvodb, + mergedvodb_path + + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue mergedvodb %" PRId64, minidvodb_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + + } + +psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + + + + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", false, false); + psString query = NULL; + + query = psStringCopy("UPDATE mergedvodbRun SET"); + + int cnt = 0; + psString comma = ","; + if (state) { + psStringAppend(&query, " state = '%s'", state); + cnt++; + } + + if (mergedvodb_path) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " mergedvodb_path = '%s'", mergedvodb_path); + cnt++; + } + + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + + return true; +} + +static bool pendingmergeMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "mergedvodbRun.minidvodb_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = NULL; + query = pxDataGet("mergetool_find_pending_merge.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "mergedvodbPendingMerge", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addmergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_LOOKUP_U64(merge_id, config->args, "-merge_id", true, false); + PXOPT_LOOKUP_STR(mergedvodb, config->args, "-mergedvodb", true, false); + + // optional + +//PXOPT_LOOKUP_U64(merge_order, config->args, "-merge_order", false, false); + PXOPT_LOOKUP_F32(dtime_merge, config->args, "-dtime_merge", false, false); + PXOPT_LOOKUP_F32(dtime_verify, config->args, "-dtime_verify", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + + PXOPT_LOOKUP_TIME(epoch, config->args, "-epoch", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psString query = pxDataGet("mergetool_find_pendingmergeprocess.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClaus = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClaus); + psFree(whereClaus); + } + psFree(where); + psTrace("mergetool.c", PS_LOG_INFO, "query is:\n%s", query); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + +psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("mergetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + mergedvodbRunRow *pendingRow = mergedvodbRunObjectFromMetadata(output->data[0]); + psFree(output); + mergedvodbProcessedRow *row = mergedvodbProcessedRowAlloc( + pendingRow->merge_id, + 0, + dtime_verify ? dtime_verify : 0, + dtime_merge ? dtime_merge : 0, + dtime_script ? dtime_script : 0, + epoch, + fault + ); + + if (!mergedvodbProcessedInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + psFree(pendingRow); + return false; + } + +//this finds the # of merged things (for the merge order) + psString query3 = NULL; + psStringAppend(&query3, "select count(*) from mergedvodbRun join mergedvodbProcessed using (merge_id) where state = 'full' and mergedvodb = '%s';", mergedvodb); + + if (!p_psDBRunQuery(config->dbh, query3)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query3); + return false; + } + psArray *output2 = p_psDBFetchResult(config->dbh); + if (!output2) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output2)) { + psTrace("mergetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + bool status; + psS64 m_order = psMetadataLookupS64(&status, output2->data[0], "count(*)"); + if (!status) { + + psAbort("failed to lookup value for count column"); + return false; + } + psString final = NULL; + psStringAppend(&final, "%" PRIu64, m_order); + //return false; + psFree(query3); + psFree(output2); + + + + //update the merge_order + + psString query4 = NULL; + psStringAppend(&query4, "update mergedvodbProcessed set merge_order = %"PRIu64,m_order); + psStringAppend(&query4," where merge_id = %" PRIu64, merge_id); + //printf("%s", query4); + if (!p_psDBRunQuery(config->dbh, query4)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query4); + return false; + } + // +psFree(query4); + +// since there is only one exp per 'new' set mergedvodbRun.state = 'full' + + psString query2 = NULL ; + + if (!fault) { + psStringAppend(&query2, "UPDATE mergedvodbRun SET state = 'full' WHERE merge_id = %'" PRIu64, row->merge_id); + + if (!p_psDBRunQuery(config->dbh, query2)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query2); + return false; + } + + } + + + + + + + psFree(row); + psFree(pendingRow); + + + + + + //commit the changes + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + + + //print the merge_order (why not!) + printf("%s", final); + psFree(final); + + + + + + + + + + + + return true; +} + + +static bool listmergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "mergedvodbRun.minidvodb_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psString query = NULL; + query = pxDataGet("mergetool_find_processed.sql"); + +if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + + + // we either add AND (condition) or WHERE (condition): + if (psListLength(where->list) && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " AND fault != 0"); + } + if (psListLength(where->list) && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND fault = 0"); + } + if (!psListLength(where->list) && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " WHERE fault != 0"); + } + if (!psListLength(where->list) && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " WHERE fault = 0"); + } + +psFree(where); + + // order by merge_id + psStringAppend(&query, " ORDER BY merge_id"); + + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + + + return true; +} + + +static bool revertmergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "mergedvodbRun.fault", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = NULL; + query = pxDataGet("mergetool_revertmergedvodbprocessed.sql"); + +if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + + return true; +} + +static bool updatemergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-merge_id", true, false); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbProcessed.merge_id", "=="); + PXOPT_LOOKUP_F32(dtime_merge, config->args, "-set_dtime_merge", false, false); + PXOPT_LOOKUP_F32(dtime_verify, config->args, "-set_dtime_verify", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-set_dtime_script", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psString query = psStringCopy("UPDATE mergedvodbProcessed JOIN mergedvodbRun USING (merge_id) SET "); +int cnt = 0; + psString comma = ","; + if (fault) { + psStringAppend(&query, " fault = %d", fault); + cnt++; + } + + + + if (dtime_verify) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_verify = %f", dtime_verify); + cnt++; + } + + if (dtime_merge) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_merge = %f", dtime_merge); + cnt++; + } + +if (dtime_script) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_script = %f", dtime_script); + cnt++; + } +if (state) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " state = '%s'", state); + cnt++; + } + + + + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error \n%s\n",query ); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + return true; +} + +static bool definebyquerymergecopyMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + pxAddLabelSearchArgs (config, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); // define using camRun label + + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(destination_host, config->args, "-set_destination_host", false, false); + PXOPT_LOOKUP_STR(mergedvodb_rsync_path, config->args, "-set_mergedvodb_rsync_path", false, false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(last_merged, config->args, "-last_merged", false); + // prevent queueing an addRun if a given exposure has already been added to + // the given dvo database + psString dvodb_string = NULL; + psString bare_query = NULL; + if (destination_host) { + psTrace("mergedvodbtool.c", PS_LOG_INFO, "destination_host argument found (%s) using mergedvodbtool_find_merge_id_dvo.sql\n", destination_host); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("mergedvodbtool_find_merge_id_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "mergedvodbCopy.destination_host = '%s'", destination_host); + } else { + psError(PS_ERR_UNKNOWN, false, "cannot queue mergedvodbcopy run without a defined destination_host"); + + return false; + } + + if (!bare_query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + psFree(where); + return false; + } + // Take the bare query and add the dvodb selector + psString query = NULL; + psStringAppend(&query, bare_query, dvodb_string); + psFree(dvodb_string); + psFree(bare_query); + + // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; + } + if (last_merged) { + psStringAppend(&query, " ORDER BY merge_order DESC LIMIT 1"); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "mergedvodbCopy", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + // loop over our list of camRun rows to check the supplied and selected dvodb and workdir values: + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + mergedvodbRunRow *row = mergedvodbRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into mergedvodbRun"); + psFree(output); + return false; + } + + if (!mergedvodb_rsync_path) { + psError(PS_ERR_UNKNOWN, false, "cannot queue mergedvodbcopy run without a defined mergedvodb_rsync_path: merge_id %" PRId64, row->merge_id); + psFree(output); + return false; + } + if (!destination_host) { + psError(PS_ERR_UNKNOWN, false, "cannot queue mergedvodbcopy run without a defined destination_host: merge_id %" PRId64, row->merge_id); + psFree(output); + return false; + } + + psFree(row); + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + mergedvodbRunRow *row = mergedvodbRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByMergeID(config, + row->merge_id, + destination_host ? destination_host : "NULL", + mergedvodb_rsync_path ?mergedvodb_rsync_path : "NULL" + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue merge_id: %" PRId64, row->merge_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool listmergedvodbcopyMode(pxConfig *config) { + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbCopy.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodbcopy_id", "mergedvodbCopy.mergedvodbcopy_id", "=="); + PXOPT_COPY_STR(config->args, where, "-destination_host", "mergedvodbCopy.destination_host", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pending, config->args, "-pending", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("mergedvodbtool_find_mergedvodbcopy.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + +if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + +// we either add AND (condition) or WHERE (condition): + if (where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault != 0"); + } + if (where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault = 0"); + } + if (!where->list && faulted) { + + // list only faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault != 0"); + } + if (where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault = 0"); + } + if (!where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " WHERE mergedvodbCopy.fault != 0"); + } + if (!where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " WHERE mergedvodbCopy.fault = 0"); + } + psFree(where); + + if (pending) { + //add the cuts for pending (state new, no faults) + psStringAppend(&query, " %s", " AND mergedvodbCopy.state = 'new' AND mergedvodbCopy.fault = 0 AND mergedvodbCopy.destination_host IS NOT NULL AND mergedvodbCopy.mergedvodb_rsync_path IS NOT NULL"); + } + + + // order by epoch + psStringAppend(&query, " ORDER BY mergedvodbcopy_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error %s ", query); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "mergedvodbCopy", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + +return true; +} + +static bool revertmergedvodbcopyMode(pxConfig *config) { + psMetadata *where = psMetadataAlloc(); + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_COPY_S64(config->args, where, "-mergedvodbcopy_id", "mergedvodbCopy.mergedvodbcopy_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbCopy.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-destination_host", "destination_host", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "mergedvodbCopy.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = pxDataGet("mergedvodbtool_revertmergedvodbcopy.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + } + psError(PS_ERR_UNKNOWN, false, "database error %s", query ); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool updatemergedvodbcopyMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_U64(mergedvodbcopy_id, config->args, "-mergedvodbcopy_id", false, false); + PXOPT_LOOKUP_U64(merge_id, config->args, "-merge_id", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + PXOPT_LOOKUP_STR(host, config->args, "-host", false, false); + PXOPT_LOOKUP_STR(rsync_path, config->args, "-mergedvodb_rsync_path", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + PXOPT_LOOKUP_S16(set_fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_STR(set_rsync_path, config->args, "-set_mergedvodb_rsync_path", false, false); + PXOPT_LOOKUP_STR(set_host, config->args, "-set_destination_host", false, false); + PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_F32(dtime, config->args, "-set_dtime", false, false); + PXOPT_COPY_S64(config->args, where, "-mergedvodbcopy_id", "mergedvodbCopy.mergedvodbcopy_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbCopy.merge_id", "=="); + + PXOPT_COPY_STR(config->args, where, "-state", "mergedvodbCopy.state", "=="); + PXOPT_COPY_STR(config->args, where, "-host", "mergedvodbCopy.destination_host", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb_rsync_path", "mergedvodbCopy.mergedvodb_rsync_path", "=="); + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE mergedvodbCopy JOIN mergedvodbRun USING (merge_id) SET "); + int cnt = 0; + psString comma = ","; + if (set_fault) { + psStringAppend(&query, " mergedvodbCopy.fault = %d", set_fault); + cnt++; + } + if (set_rsync_path) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " mergedvodbCopy.mergedvodb_rsync_path = '%s'" , set_rsync_path); + cnt++; + } + + if (set_host) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " mergedvodbCopy.destination_host = '%s'" , set_host); + cnt++; + } + + + if (set_state) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " mergedvodbCopy.state = '%s'" , set_state); + cnt++; + } + + if (dtime) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " mergedvodbCopy.dtime = %f", dtime); + cnt++; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + return true; +} + + + Index: trunk/ippTools/src/mergetool.h =================================================================== --- trunk/ippTools/src/mergetool.h (revision 33030) +++ trunk/ippTools/src/mergetool.h (revision 33030) @@ -0,0 +1,41 @@ +/* + * mergetool.h + * + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef MERGETOOL_H +#define MERGETOOL_H 1 + +#include "pxtools.h" + +typedef enum { + MERGETOOL_MODE_NONE = 0x0, + MERGETOOL_MODE_DEFINEBYQUERY, + MERGETOOL_MODE_UPDATERUN, + MERGETOOL_MODE_PENDINGMERGE, + MERGETOOL_MODE_ADDMERGED, + MERGETOOL_MODE_LISTMERGED, + MERGETOOL_MODE_REVERTMERGED, + MERGETOOL_MODE_UPDATEMERGED, + MERGETOOL_MODE_DEFINEBYQUERYMERGECOPY, + MERGETOOL_MODE_LISTMERGEDVODBCOPY, + MERGETOOL_MODE_REVERTMERGEDVODBCOPY, + MERGETOOL_MODE_UPDATEMERGEDVODBCOPY +} mergetoolMode; + +pxConfig *mergetoolConfig(pxConfig *config, int argc, char **argv); + +#endif // MERGETOOL_H Index: trunk/ippTools/src/mergetoolConfig.c =================================================================== --- trunk/ippTools/src/mergetoolConfig.c (revision 33030) +++ trunk/ippTools/src/mergetoolConfig.c (revision 33030) @@ -0,0 +1,196 @@ +/* + * addtoolConfig.c + * + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include + +#include "pxtools.h" +#include "pxmerge.h" +#include "mergetool.h" + +pxConfig *mergetoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + // pxcamSetSearchArgs(definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-mergedvodb", PS_META_DUPLICATE_OK, "search by mergedvodbRun.mergedvodb and use as mergedvodb", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-minidvodb_group", PS_META_DUPLICATE_OK, "search by minidvodbRun.minidvodb_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define workdir", NULL); + // psMetadataAddStr(definebyqueryArge, PS_LIST_TAIL, "-set_mergedvodb", 0, "define mergedvodb"); + + // psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psMetadata *updaterunArgs = psMetadataAlloc(); + + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb", 0); + psMetadataAddU64(updaterunArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "change the state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define workdir", NULL); + + + psMetadata *pendingmergeArgs = psMetadataAlloc(); + psMetadataAddU64(pendingmergeArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + psMetadataAddStr(pendingmergeArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", NULL); + psMetadataAddStr(pendingmergeArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb", NULL); + + psMetadataAddU64(pendingmergeArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(pendingmergeArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + + + psMetadata *addmergedArgs = psMetadataAlloc(); + psMetadataAddU64(addmergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id (required)", 0); + psMetadataAddStr(addmergedArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb (required)", 0); + psMetadataAddF32(addmergedArgs, PS_LIST_TAIL, "-dtime_verify", 0, "define elapsed time for DVO verify (seconds)", NAN); + psMetadataAddF32(addmergedArgs, PS_LIST_TAIL, "-dtime_merge", 0, "define elapsed time for DVO merge (seconds)", NAN); + psMetadataAddF32(addmergedArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time for script (seconds)", NAN); + psMetadataAddTime(addmergedArgs, PS_LIST_TAIL, "-epoch", 0, "time merge is finished", NULL); + psMetadataAddS16(addmergedArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + + + + psMetadata *listmergedArgs = psMetadataAlloc(); + psMetadataAddU64(listmergedArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + psMetadataAddStr(listmergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", NULL); + psMetadataAddStr(listmergedArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by minidvodbRun.minidvodb_group", NULL); + psMetadataAddU64(listmergedArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(listmergedArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + psMetadataAddBool(listmergedArgs, PS_LIST_TAIL, "-faulted", 0, "limit to faulted state", false); + + + psMetadata *revertmergedArgs = psMetadataAlloc(); + psMetadataAddU64(revertmergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddS16(revertmergedArgs, PS_LIST_TAIL, "-fault", 0, "search by minidvodbCopyfault", 0); + psMetadataAddStr(revertmergedArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb", NULL); + + psMetadata *updatemergedArgs = psMetadataAlloc(); + + psMetadataAddU64(updatemergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddS16(updatemergedArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); + psMetadataAddF32(updatemergedArgs, PS_LIST_TAIL, "-set_dtime_verify", 0, "define elapsed time for DVO verify (seconds)", 0); + psMetadataAddF32(updatemergedArgs, PS_LIST_TAIL, "-set_dtime_merge", 0, "define elapsed time for DVO merge (seconds)", 0); + psMetadataAddF32(updatemergedArgs, PS_LIST_TAIL, "-set_dtime_script", 0, "define elapsed time for script (seconds)", 0); + psMetadataAddStr(updatemergedArgs, PS_LIST_TAIL, "-set_state", 0, "change the state", NULL); + // -definebyquerymergecopy + psMetadata *definebyquerymergecopyArgs = psMetadataAlloc(); + psMetadataAddS64(definebyquerymergecopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(definebyquerymergecopyArgs, PS_LIST_TAIL, "-mergedvodb", PS_META_DUPLICATE_OK, "search by mergedvodb", NULL); + psMetadataAddStr(definebyquerymergecopyArgs, PS_LIST_TAIL, "-set_mergedvodb_rsync_path", 0, "define workdir", NULL); + psMetadataAddStr(definebyquerymergecopyArgs, PS_LIST_TAIL, "-set_destination_host", 0, "set destination host", NULL); + psMetadataAddBool(definebyquerymergecopyArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyquerymergecopyArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definebyquerymergecopyArgs, PS_LIST_TAIL, "-last_merged", 0, "if multiple minidvodbs have been dvomerged between rsyncs, then only queue the most recent merged dvodb for copying", false); + + psMetadata *listmergedvodbcopyArgs = psMetadataAlloc(); + psMetadataAddU64(listmergedvodbcopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by mergedvodb_id", 0); + psMetadataAddStr(listmergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodbcopy_id", 0, "search by mergedvodbcopy_id", NULL); + psMetadataAddStr(listmergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodbRun.mergedvodb", NULL); + psMetadataAddStr(listmergedvodbcopyArgs, PS_LIST_TAIL, "-destination_host", 0, "search by mergedvodbCopy.destination_host", NULL); + psMetadataAddBool(listmergedvodbcopyArgs, PS_LIST_TAIL, "-pending", 0, "limit to pending items", false); + psMetadataAddU64(listmergedvodbcopyArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(listmergedvodbcopyArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + psMetadataAddBool(listmergedvodbcopyArgs, PS_LIST_TAIL, "-faulted", 0, "limit to faulted state", false); + + psMetadata *revertmergedvodbcopyArgs = psMetadataAlloc(); + psMetadataAddU64(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodbcopy_id", 0, "search by mergedvodbcopy_id", 0); + psMetadataAddU64(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-destination_host", 0, "search by destination_host", NULL); + psMetadataAddS16(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-fault", 0, "search by mergedvodbCopyfault", 0); + psMetadata *updatemergedvodbcopyArgs = psMetadataAlloc(); + psMetadataAddU64(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodbcopy_id", 0, "search by mergedvodbcopy_id", 0); + psMetadataAddU64(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-host", 0, "search by host", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodb_rsync_path", 0, "search by rsync", NULL); + psMetadataAddS16(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + + + psMetadataAddS16(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); + + psMetadataAddF32(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_dtime", 0, "set elapsed time for transfer", 0); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_mergedvodb_rsync_path", 0, "change the mergedvodb_rsync_path", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_destination_host", 0, "change the host", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_state", 0, "change the state", NULL); + + + + + + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from minindvodbRun", MERGETOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "update existing mergedvodbRuns", MERGETOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingmerge", "list pending mergedvodbRuns", MERGETOOL_MODE_PENDINGMERGE, pendingmergeArgs); + PXOPT_ADD_MODE("-addmerged", "add processed mergedvodbRun", MERGETOOL_MODE_ADDMERGED, addmergedArgs); + PXOPT_ADD_MODE("-listmerged","list merged dvodbs", MERGETOOL_MODE_LISTMERGED, listmergedArgs); + PXOPT_ADD_MODE("-revertmerged","revert merged dvodbs", MERGETOOL_MODE_REVERTMERGED, revertmergedArgs); + PXOPT_ADD_MODE("-updatemerged","change merged dvodb properties",MERGETOOL_MODE_UPDATEMERGED, updatemergedArgs); + PXOPT_ADD_MODE("-definebyquerymergecopy", "create runs from mergedvodbRun", MERGETOOL_MODE_DEFINEBYQUERYMERGECOPY, definebyquerymergecopyArgs); + PXOPT_ADD_MODE("-listmergedvodbcopy","list copy mergedvodbs", MERGETOOL_MODE_LISTMERGEDVODBCOPY, listmergedvodbcopyArgs); + PXOPT_ADD_MODE("-revertmergedvodbcopy","revert copy mergedvobs",MERGETOOL_MODE_REVERTMERGEDVODBCOPY, revertmergedvodbcopyArgs); + PXOPT_ADD_MODE("-updatemergedvodbcopy","change mergedvodb copy properties",MERGETOOL_MODE_UPDATEMERGEDVODBCOPY, updatemergedvodbcopyArgs); + + + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: trunk/ippTools/src/minidvodbtool.c =================================================================== --- trunk/ippTools/src/minidvodbtool.c (revision 33024) +++ trunk/ippTools/src/minidvodbtool.c (revision 33030) @@ -89,5 +89,5 @@ psMetadata *where = psMetadataAlloc(); pxcamGetSearchArgs (config, where); - PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodbRun_id", "=="); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodb_id", "=="); pxAddLabelSearchArgs (config, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "=="); // define using camRun label Index: trunk/ippTools/src/pxmerge.c =================================================================== --- trunk/ippTools/src/pxmerge.c (revision 33030) +++ trunk/ippTools/src/pxmerge.c (revision 33030) @@ -0,0 +1,97 @@ +/* + * pxadd.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxmerge.h" + +bool pxmergeQueueByMinidvodbID(pxConfig *config, + psS64 minidvodb_id, + char *mergedvodb, + char *mergedvodb_path + ) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("mergetool_queue_merge_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + } + if (!p_psDBRunQueryF(config->dbh,"INSERT INTO mergedvodbRun SELECT 0, minidvodb_id, '%s', '%s', 'new', NULL FROM minidvodbRun WHERE minidvodbRun.minidvodb_id = %lld and minidvodbRun.state = 'merged'", mergedvodb, mergedvodb_path, (long long) minidvodb_id)) + { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + return false; + } + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row \n%s", query); + return false; + } + return true; +} + +bool pxaddQueueByMergeID(pxConfig *config, + psS64 merge_id, + char *destination_host, + char *mergedvodb_rsync_path + ) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("mergetool_queue_mergecopy_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + } + + if (!p_psDBRunQueryF(config->dbh,"INSERT INTO mergedvodbCopy SELECT 0, merge_id, '%s', '%s', 0, 'new', NULL, 0 FROM mergedvodbRun WHERE mergedvodbRun.merge_id = %lld and (mergedvodbRun.state = 'merged' or mergedvodbRun.state = 'full')", mergedvodb_rsync_path, destination_host, (long long) merge_id)) + + { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row \n%s", query); + return false; + } + + return true; +} Index: trunk/ippTools/src/pxmerge.h =================================================================== --- trunk/ippTools/src/pxmerge.h (revision 33030) +++ trunk/ippTools/src/pxmerge.h (revision 33030) @@ -0,0 +1,41 @@ +/* + * pxadd.h + * + * Copyright (C) 2009 Joshua Hoblitt, Chris Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXMERGE_H +#define PXMERGE_H 1 + +#include + +#include "pxtools.h" + + +bool pxmergeQueueByMinidvodbID(pxConfig *config, + psS64 minidvodb_id, + char *mergedvodbb, + char *mergedvodb_path + ); + +bool pxaddQueueByMergeID(pxConfig *config, + psS64 merge_id, + char *destination_host, + char *mergedvodb_rsync_path + ); + + +#endif // PXMERGE_H