Index: /trunk/ippTools/src/Makefile.am =================================================================== --- /trunk/ippTools/src/Makefile.am (revision 10040) +++ /trunk/ippTools/src/Makefile.am (revision 10041) @@ -8,5 +8,4 @@ pxinject \ pzgetexp \ - pzgetfile \ pzgetimfiles \ pztool @@ -22,5 +21,4 @@ pxinject.h \ pzgetexp.h \ - pzgetfile.h \ pzgetimfiles.h \ pztool.h @@ -86,10 +84,4 @@ pzgetimfilesConfig.c -pzgetfile_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) $(pxtools_CFLAGS) -pzgetfile_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la -pzgetfile_SOURCES = \ - pzgetfile.c \ - pzgetfileConfig.c - dettool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) $(pxtools_CFLAGS) dettool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la Index: unk/ippTools/src/pzgetfile.c =================================================================== --- /trunk/ippTools/src/pzgetfile.c (revision 10040) +++ (revision ) @@ -1,509 +1,0 @@ -/* - * pzgetfile.c - * - * Copyright (C) 2006 Joshua Hoblitt - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along with - * program; see the file COPYING. If not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include -#include - -#include "pxtools.h" -#include "pzgetfile.h" - -#define FILESET_LS_CMD "dsfilesetls" - -static bool go (pxConfig *config); -static psArray *parseFiles(pxConfig *config, const char *str); - -int main(int argc, char **argv) -{ - psLibInit(NULL); - - pxConfig *config = pzgetfileConfig(NULL, argc, argv); - - if (!go(config)) { - goto FAIL; - } - - psFree(config); - pmConfigDone(); - psLibFinalize(); - - exit(EXIT_SUCCESS); - -FAIL: - psFree(config); - pmConfigDone(); - psLibFinalize(); - - exit(EXIT_FAILURE); -} - -static bool go(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, NULL); - - bool status = false; - psString uri = psMetadataLookupStr(&status, config->args, "-uri"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); - return false; - } - if (!uri) { - psError(PS_ERR_UNKNOWN, true, "-uri is required"); - return false; - } - - psString filesetid = psMetadataLookupStr(&status, config->args, "-filesetid"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filesetid"); - return false; - } - if (!filesetid) { - psError(PS_ERR_UNKNOWN, true, "-filesetid is required"); - return false; - } - - psString camera = psMetadataLookupStr(&status, config->args, "-inst"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst"); - return NULL; - } - if (!camera) { - psError(PS_ERR_UNKNOWN, true, "-inst is required"); - return NULL; - } - - psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope"); - return NULL; - } - if (!telescope) { - psError(PS_ERR_UNKNOWN, true, "-telescope is required"); - return NULL; - } - - // invoke dsfilesetls - psString cmd = NULL; - psStringAppend(&cmd, "%s --uri %s", FILESET_LS_CMD, uri); - - psTrace("pzgetimfiles", PS_LOG_INFO, "cmd is: %s\n", cmd); - - FILE *output = popen(cmd, "r"); - psFree(cmd); - - if (!output) { - psError(PS_ERR_UNKNOWN, true, "popen() failed"); - return false; - } - - psString cmdOutput = psSlurpFile(output); - int exitStatus = pclose(output); - - if (exitStatus != 0) { - psError(PS_ERR_UNKNOWN, true, "%s failed with exit status %d", - FILESET_LS_CMD, exitStatus); - return false; - } - - // prase output of dsfilesetls - psArray *newImfiles = parseFiles(config, cmdOutput); - if (!newImfiles) { - // XXX not nessicarily an error - psError(PS_ERR_UNKNOWN, true, "no new files/imfiles"); - psFree(cmdOutput); - return false; - } - psFree(cmdOutput); - - // save the number of new Imfiles; - long imfiles = psArrayLength(newImfiles); - - // start a transaction so it's all rows or nothing - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - // if the fileset was empty (no files) then we can bail out early - if (imfiles == 0) { - psFree(newImfiles); - - char *query = - "UPDATE summitExp" - " SET imfiles = %d" - " WHERE exp_id = '%s'" - " AND camera = '%s'" - " AND telescope = '%s'"; - if (!p_psDBRunQuery(config->dbh, query, imfiles, filesetid, camera, telescope)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - // remove the pzPendingExp entry for this exp (fileset) - { - char *query = - "DELETE FROM pzPendingExp" - " WHERE" - " exp_id = '%s'" - " AND camera = '%s'" - " AND telescope = '%s'"; - if (!p_psDBRunQuery(config->dbh, query, filesetid, camera, telescope)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - } - - // point of no return - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - return true; - } - - // try not to insert duplicate pzPendingExp/pzPendingImfile entries - - // create a temp table - { - char *query = - "CREATE TEMPORARY TABLE incoming (exp_id VARCHAR(64), camera VARCHAR(64), telescope VARCHAR(64), bytes INT, md5sum VARCHAR(32), class VARCHAR(64), class_id VARCHAR(64), uri VARCHAR(255), PRIMARY KEY(exp_id, camera, telescope, class, class_id)) ENGINE=MEMORY"; - - if (!p_psDBRunQuery(config->dbh, query)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(newImfiles); - return false; - } - } - - // load the imfiles (files) into the temp table - { - char *query = "INSERT INTO incoming (exp_id, camera, telescope, bytes, md5sum, class, class_id, uri) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"; - - long inserted = p_psDBRunQueryPrepared(config->dbh, newImfiles, query); - if (inserted < 0) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(newImfiles); - return false; - } - // sanity check that we actually inserted something - if (inserted == 0) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error -- we should have inserted at least one row"); - psFree(newImfiles); - return false; - } - } - - psFree(newImfiles); - - // queue the imfiles (files) in pzPendingImfile - { - char *query = - "INSERT INTO pzPendingImfile" - " SElECT" - " incoming.exp_id," - " incoming.camera," - " incoming.telescope," - " incoming.class," - " incoming.class_id," - " pzPendingExp.exp_tag" - " FROM incoming" - " JOIN pzPendingExp" - " USING(exp_id, camera, telescope)" - " LEFT JOIN summitImfile" - " USING(exp_id, camera, telescope, class, class_id)" - " WHERE" - " summitImfile.exp_id is NULL" - " AND summitImfile.camera is NULL" - " AND summitImfile.telescope is NULL" - " AND summitImfile.class is NULL" - " AND summitImfile.class_id is NULL"; - - if (!p_psDBRunQuery(config->dbh, query)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - } - - // move imfiles (files) from the temp table into summitImfiles - { - char *query = - "INSERT INTO summitImfile" - " SElECT" - " incoming.exp_id," - " incoming.camera," - " incoming.telescope," - " incoming.bytes," - " incoming.md5sum," - " incoming.class," - " incoming.class_id," - " incoming.uri" - " FROM incoming" - " LEFT JOIN summitImfile" - " USING(exp_id, camera, telescope, class, class_id)" - " WHERE" - " summitImfile.exp_id is NULL" - " AND summitImfile.camera is NULL" - " AND summitImfile.telescope is NULL" - " AND summitImfile.class is NULL" - " AND summitImfile.class_id is NULL"; - - if (!p_psDBRunQuery(config->dbh, query)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - } - - // update summitExp.imfiles (should have been NULL) - { - char *query = - "UPDATE summitExp" - " SET imfiles = (SELECT COUNT(*) FROM summitImfile" - " WHERE" - " exp_id = '%s'" - " AND camera = '%s'" - " AND telescope = '%s'" - ")" - " WHERE" - " exp_id = '%s'" - " AND camera = '%s'" - " AND telescope = '%s'"; - if (!p_psDBRunQuery(config->dbh, query, filesetid, camera, telescope, filesetid, camera, telescope)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - } - - // remove the pzPendingExp entry for this exp (fileset) - { - char *query = - "DELETE FROM pzPendingExp" - " WHERE" - " exp_id = '%s'" - " AND camera = '%s'" - " AND telescope = '%s'"; - if (!p_psDBRunQuery(config->dbh, query, filesetid, camera, telescope)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - } - - // point of no return - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - return true; -} - -static psArray *parseFiles(pxConfig *config, const char *str) -{ - PS_ASSERT_PTR_NON_NULL(config, NULL); - PS_ASSERT_PTR_NON_NULL(str, NULL); - - // these are constants for all records parsed -- look them up before we do - // any work - bool status = false; - psString exp_id = psMetadataLookupStr(&status, config->args, "-filesetid"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for '-filesetid'"); - return NULL; - } - if (!exp_id) { - psError(PS_ERR_UNKNOWN, true, "-filesetid is required"); - return NULL; - } - - psString camera = psMetadataLookupStr(&status, config->args, "-inst"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst"); - return NULL; - } - if (!camera) { - psError(PS_ERR_UNKNOWN, true, "-inst is required"); - return NULL; - } - - psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope"); - return NULL; - } - if (!telescope) { - psError(PS_ERR_UNKNOWN, true, "-telescope is required"); - return NULL; - } - - // split the string into lines - psList *doc = psStringSplit(str, "\n", false); - - psListIterator *lineCursor = psListIteratorAlloc(doc, 0, false); - - psArray *pzPendingImfiles = psArrayAllocEmpty(psListLength(doc)); - psString line; - while ((line = psListGetAndIncrement(lineCursor))) { - psTrace("pzgetimfiles", PS_LOG_INFO, "parsing line: %s\n", line); - - // split line into tokens - psList *tokens = psStringSplit(line, " ", false); - - // check to see if this line is a comment (or if the first token is - // NULL) - if (!psListGet(tokens, 0) || *((char *)psListGet(tokens, 0)) == '#') { - psFree(tokens); - continue; - } - - // check that we have the right number of tokens - // print "# uri fileid bytes md5sum type \n"; - if (psListLength(tokens) != 5) { - psError(PS_ERR_UNKNOWN, true, "invalid line format: %s", line); - return false; - } - - // find the values of interest - psListIterator *tokenCursor = psListIteratorAlloc(tokens, 0, false); - char *uri = psListGetAndIncrement(tokenCursor); - char *class_id = psListGetAndIncrement(tokenCursor); // fileid - char *bytes = psListGetAndIncrement(tokenCursor); // bytes - char *md5sum = psListGetAndIncrement(tokenCursor); // md5sum - char *class = psListGetAndIncrement(tokenCursor); // type - - // create a new metadata to represent this line and it's values - psMetadata *md = psMetadataAlloc(); - if (!psMetadataAddStr(md, PS_LIST_TAIL, "exp_id", 0, NULL, exp_id)) { - psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); - psFree(md); - psFree(tokenCursor); - psFree(tokens); - return NULL; - } - if (!psMetadataAddStr(md, PS_LIST_TAIL, "camera", 0, NULL, camera)) { - psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); - psFree(md); - psFree(tokenCursor); - psFree(tokens); - return NULL; - } - if (!psMetadataAddStr(md, PS_LIST_TAIL, "telescope", 0, NULL, telescope)) { - psError(PS_ERR_UNKNOWN, false, "failed to add item telescope"); - psFree(md); - psFree(tokenCursor); - psFree(tokens); - return NULL; - } - if (!psMetadataAddS32(md, PS_LIST_TAIL, "bytes", 0, NULL, (psS32)atoi(bytes))) { - psError(PS_ERR_UNKNOWN, false, "failed to add item bytes"); - psFree(md); - psFree(tokenCursor); - psFree(tokens); - return NULL; - } - if (!psMetadataAddStr(md, PS_LIST_TAIL, "md5sum", 0, NULL, md5sum)) { - psError(PS_ERR_UNKNOWN, false, "failed to add item md5sum"); - psFree(md); - psFree(tokenCursor); - psFree(tokens); - return NULL; - } - if (!psMetadataAddStr(md, PS_LIST_TAIL, "class", 0, NULL, class)) { - psError(PS_ERR_UNKNOWN, false, "failed to add item class"); - psFree(md); - psFree(tokenCursor); - psFree(tokens); - return NULL; - } - if (!psMetadataAddStr(md, PS_LIST_TAIL, "class_id", 0, NULL, class_id)) { - psError(PS_ERR_UNKNOWN, false, "failed to add item class_id"); - psFree(md); - psFree(tokenCursor); - psFree(tokens); - return NULL; - } - if (!psMetadataAddStr(md, PS_LIST_TAIL, "uri", 0, NULL, uri)) { - psError(PS_ERR_UNKNOWN, false, "failed to add item uri"); - psFree(md); - psFree(tokenCursor); - psFree(tokens); - return NULL; - } - - // must be freed after the new metadata is built -- holds the strings - psFree(tokenCursor); - psFree(tokens); - - // debugging - if (psTraceGetLevel("pzgetimfiles") == PS_LOG_INFO) { - psString doc = psMetadataConfigFormat(md); - psTrace("pzgetimfiles", PS_LOG_INFO, "parsed line as:\n %s\n", doc); - psFree(doc); - } - - psArrayAdd(pzPendingImfiles, 0, md); - - psFree(md); - } - - psFree(lineCursor); - psFree(doc); - - return pzPendingImfiles;; -} Index: unk/ippTools/src/pzgetfile.h =================================================================== --- /trunk/ippTools/src/pzgetfile.h (revision 10040) +++ (revision ) @@ -1,27 +1,0 @@ -/* - * pzgetfile.h - * - * Copyright (C) 2006 Joshua Hoblitt - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along with - * program; see the file COPYING. If not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef PZGETFILE_H -#define PZGETFILE 1 - -#include "pxtools.h" - -pxConfig *pzgetfileConfig(pxConfig *config, int argc, char **argv); - -#endif // PZGETFILE_H Index: unk/ippTools/src/pzgetfileConfig.c =================================================================== --- /trunk/ippTools/src/pzgetfileConfig.c (revision 10040) +++ (revision ) @@ -1,81 +1,0 @@ -/* - * pzgetfileConfig.c - * - * Copyright (C) 2006 Joshua Hoblitt - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along with - * program; see the file COPYING. If not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include -#include - -#include "pxtools.h" - -pxConfig *pzgetfileConfig(pxConfig *config, int argc, char **argv) { - if (!config) { - config = pxConfigAlloc(); - } - - config->modules = pmConfigRead(&argc, argv); - - if (! config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); - exit(EXIT_FAILURE); - } - - psMetadata *args = psMetadataAlloc(); - psMetadataAddStr(args , PS_LIST_TAIL, "-uri", 0, - "DataStore FileSet URI", ""); - psMetadataAddStr(args , PS_LIST_TAIL, "-filesetid", 0, - "FileSet ID", ""); - psMetadataAddStr(args, PS_LIST_TAIL, "-inst", 0, - "search by camera", NULL); - psMetadataAddStr(args, PS_LIST_TAIL, "-telescope", 0, - "search by telescope", NULL); - - - bool status = false; - if (!psArgumentParse(args, &argc, argv) - || argc != 1 - || strcmp(psMetadataLookupStr(&status, args, "-uri"), "") == 0 - || strcmp(psMetadataLookupStr(&status, args, "-filesetid"), "") == 0 - ) { - fprintf(stderr, "error parsing arguments\n"); - printf("\nPan-STARRS Phase Z Search Tool\n"); - printf("Usage: %s -uri -filesetid \n\n", argv[0]); - psArgumentHelp(args); - psFree(config); - exit(EXIT_FAILURE); - } - - config->args = args; - // don't free args here as it's silly to increment the ref count then - // "free" it - - // define Database handle, if used - config->dbh = pmConfigDB(config->modules); - if(!config->dbh) { - psError(PS_ERR_UNKNOWN, false, "Can't connect to db\n"); - psFree(config); - exit(EXIT_FAILURE); - } - - return config; -}