Index: /trunk/ippTools/src/pzgetimfiles.c =================================================================== --- /trunk/ippTools/src/pzgetimfiles.c (revision 10013) +++ /trunk/ippTools/src/pzgetimfiles.c (revision 10014) @@ -31,4 +31,5 @@ #define FILESET_LS_CMD "dsfilesetls" +static bool go (pxConfig *config); static psArray *parseFiles(pxConfig *config, const char *str); @@ -37,7 +38,43 @@ pxConfig *config = pzgetimfilesConfig(NULL, argc, argv); - // invoke dsfilesetls + if (!go(config)) { + goto FAIL; + } + + psFree(config); + exit(EXIT_SUCCESS); + +FAIL: + psFree(config); + + exit(EXIT_FAILURE); +} + +static bool go (pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + bool status = false; psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + + psString filesetid = psMetadataLookupStr(&status, config->args, "-filesetid"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filesetid"); + return false; + } + if (!filesetid) { + psError(PS_ERR_UNKNOWN, true, "-filesetid is required"); + return false; + } + + // invoke dsfilesetls psString cmd = NULL; psStringAppend(&cmd, "%s --uri %s", FILESET_LS_CMD, uri); @@ -51,5 +88,5 @@ if (!output) { psError(PS_ERR_UNKNOWN, true, "popen() failed"); - goto FAIL; + return false; } @@ -60,5 +97,5 @@ psError(PS_ERR_UNKNOWN, true, "%s failed with exit status %d", FILESET_LS_CMD, exitStatus); - goto FAIL; + return false; } @@ -68,100 +105,122 @@ // XXX not nessicarily an error psError(PS_ERR_UNKNOWN, true, "no new files/imfiles"); - goto FAIL; + psFree(cmdOutput); + return false; + } + + psFree(cmdOutput); + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; } // try not to insert duplicate pzPendingExp/pzPendingImfile entries - // XXX this will become very expensive as the number of records grows - // XXX change psDB to support inserting data with 'not exists' - psArray *pzPendingImfiles = - pzPendingImfileSelectRowObjects(config->dbh, NULL, 0); - if (pzPendingImfiles) { - for (long i = 0; i < psArrayLength(newImfiles); i++) { - pzPendingImfileRow *newImfile = newImfiles->data[i]; - for (long j = 0; j < psArrayLength(pzPendingImfiles); j++) { - pzPendingImfileRow *pendingImfile = pzPendingImfiles->data[j]; - if (strcmp(newImfile->exp_id, - pendingImfile->exp_id) == 0) { - psArrayRemoveData(newImfiles, newImfile); - // dec the counter as the array just got shorter - // and we don't want to skip elemnts - i--; - break; - } - } - } - psFree(pzPendingImfiles); - } - - // insert new entries into the database - for (long i = 0; i < psArrayLength(newImfiles); i++) { - if (!pzPendingImfileInsertObject(config->dbh, newImfiles->data[i])) { - psError(PS_ERR_UNKNOWN, false, "dbh access failed"); - goto FAIL; + + // create a temp table + { + char *query = + "CREATE TEMPORARY TABLE incoming (exp_id VARCHAR(64), camera VARCHAR(64), telescope VARCHAR(64), bytes INT, md5sum VARCHAR(32), class VARCHAR(64), class_id VARCHAR(64), uri VARCHAR(255), PRIMARY KEY(exp_id, camera, telescope, class, class_id)) ENGINE=MEMORY"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newImfiles); + return false; + } + } + + { + char *query = "INSERT INTO incoming (exp_id, camera, telescope, bytes, md5sum, class, class_id, uri) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"; + + long inserted = p_psDBRunQueryPrepared(config->dbh, newImfiles, query); + if (inserted < 0) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newImfiles); + return false; + } + // sanity check that we actually inserted something + if (inserted == 0) { + psError(PS_ERR_UNKNOWN, false, "database error -- we should have inserted at least one row"); + psFree(newImfiles); + return false; } } // save the number of new Imfiles; - long nImfiles = psArrayLength(newImfiles); + long imfiles = psArrayLength(newImfiles); psFree(newImfiles); - - // find the summitExp entry for our exp_id - psArray *summitExps = NULL; - { - psMetadata *where = psMetadataAlloc(); - bool status = false; - psString exp_id = psMetadataLookupStr(&status, config->args, - "-filesetid"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to find arg exp_id"); - psFree(where); - goto FAIL; - } - if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", 0, "==", exp_id)) { - psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); - psFree(where); - goto FAIL; - } - summitExps = summitExpSelectRowObjects(config->dbh, where, 0); - psFree(where); - if (!summitExps) { - // this REALLY should not happen - psError(PS_ERR_UNKNOWN, false, - "failed to find summitExp with exp_id %s (should not happen)" - , exp_id); - goto FAIL; - } - } - - // insert a new pzPendingExp - summitExpRow *summitExp = summitExps->data[0]; - pzPendingExpRow *pzPendingExp = pzPendingExpRowAlloc( - summitExp->exp_id, - summitExp->camera, - summitExp->telescope, - summitExp->exp_type, - nImfiles - ); - psFree(summitExps); - if (!pzPendingExp) { - psError(PS_ERR_UNKNOWN, false, "pzPendingExpRowAlloc() failed"); - goto FAIL; - } - - if (!pzPendingExpInsertObject(config->dbh, pzPendingExp)) { - psError(PS_ERR_UNKNOWN, false, "dbh access failed"); - psFree(pzPendingExp); - goto FAIL; - } - - psFree(pzPendingExp); - - psFree(config); - exit(EXIT_SUCCESS); - -FAIL: - psFree(config); - - exit(EXIT_FAILURE); + { + char *query = + "INSERT INTO pzPendingImfile" + " SElECT" + " incoming.exp_id," + " incoming.camera," + " incoming.telescope," + " incoming.class," + " incoming.class_id," + " pzPendingExp.exp_tag" + " FROM incoming" + " JOIN pzPendingExp" + " USING(exp_id, camera, telescope)" + " LEFT JOIN summitImfile" + " USING(exp_id, camera, telescope, class, class_id)" + " WHERE" + " summitImfile.exp_id is NULL" + " AND summitImfile.camera is NULL" + " AND summitImfile.telescope is NULL" + " AND summitImfile.class is NULL" + " AND summitImfile.class_id is NULL"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + + { + char *query = + "INSERT INTO summitImfile" + " SElECT" + " incoming.exp_id," + " incoming.camera," + " incoming.telescope," + " incoming.bytes," + " incoming.md5sum," + " incoming.class," + " incoming.class_id," + " incoming.uri" + " FROM incoming" + " LEFT JOIN summitImfile" + " USING(exp_id, camera, telescope, class, class_id)" + " WHERE" + " summitImfile.exp_id is NULL" + " AND summitImfile.camera is NULL" + " AND summitImfile.telescope is NULL" + " AND summitImfile.class is NULL" + " AND summitImfile.class_id is NULL"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // update summitExp.imfiles + { + char *query = "UPDATE summitExp SET imfiles = %d"; + if (!p_psDBRunQuery(config->dbh, query, imfiles)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; } @@ -171,8 +230,38 @@ PS_ASSERT_PTR_NON_NULL(str, NULL); + // these are constants for all records parsed -- look them up before we do + // any work bool status = false; - psString camera = psMetadataLookupStr(&status, config->args, "-inst"); - psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); - + psString exp_id = psMetadataLookupStr(&status, config->args, "-filesetid"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for '-filesetid'"); + return NULL; + } + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "-filesetid is required"); + return NULL; + } + + psString camera = psMetadataLookupStr(&status, config->args, "-inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst"); + return NULL; + } + if (!camera) { + psError(PS_ERR_UNKNOWN, true, "-inst is required"); + return NULL; + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope"); + return NULL; + } + if (!telescope) { + psError(PS_ERR_UNKNOWN, true, "-telescope is required"); + return NULL; + } + + // split the string into lines psList *doc = psStringSplit(str, "\n", false); @@ -182,7 +271,4 @@ psString line; while ((line = psListGetAndIncrement(lineCursor))) { - // XXX debugging - printf("-> %s\n", line); - // split line into tokens psList *tokens = psStringSplit(line, " ", false); @@ -202,4 +288,5 @@ } + // find the values of interest psListIterator *tokenCursor = psListIteratorAlloc(tokens, 0, false); char *uri = psListGetAndIncrement(tokenCursor); @@ -209,23 +296,68 @@ char *class = psListGetAndIncrement(tokenCursor); // type - bool status = false; - psString exp_id = psMetadataLookupStr(&status, config->args, - "-filesetid"); - - pzPendingImfileRow *row = pzPendingImfileRowAlloc( - exp_id, - camera, - telescope, - (psS32)atol(bytes), - md5sum, - class, - class_id, - uri - ); + // create a new metadata to represent this line and it's values + psMetadata *md = psMetadataAlloc(); + if (!psMetadataAddStr(md, PS_LIST_TAIL, "exp_id", 0, NULL, exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "camera", 0, NULL, camera)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "telescope", 0, NULL, telescope)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item telescope"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddS32(md, PS_LIST_TAIL, "bytes", 0, NULL, (psS32)atoi(bytes))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item bytes"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "md5sum", 0, NULL, md5sum)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item md5sum"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "class", 0, NULL, class)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "class_id", 0, NULL, class_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class_id"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "uri", 0, NULL, uri)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item uri"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } psFree(tokenCursor); psFree(tokens); - psArrayAdd(pzPendingImfiles, 0, row); + psArrayAdd(pzPendingImfiles, 0, md); + psMetadataConfigPrint(stdout, md); }