Index: /trunk/ippTools/src/Makefile.am =================================================================== --- /trunk/ippTools/src/Makefile.am (revision 10027) +++ /trunk/ippTools/src/Makefile.am (revision 10028) @@ -1,27 +1,28 @@ bin_PROGRAMS = \ - pxadmin \ - pztool \ + detselect \ + dettool \ p0tool \ p2tool\ p3tool\ - dettool \ - detselect \ + pxadmin \ + pxinject \ pzgetexp \ + pzgetfile \ pzgetimfiles \ - pxinject - + pztool include_HEADERS = pxtools.h noinst_HEADERS = \ - pxadmin.h \ - pzgetexp.h \ - pzgetimfiles.h \ - pztool.h \ + dettool.h \ p0tool.h \ p1tool.h \ p2tool.h \ p3tool.h \ - dettool.h \ - pxinject.h + pxadmin.h \ + pxinject.h \ + pzgetexp.h \ + pzgetfile.h \ + pzgetimfiles.h \ + pztool.h lib_LTLIBRARIES = libpxtools.la @@ -85,4 +86,10 @@ pzgetimfilesConfig.c +pzgetfile_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) $(pxtools_CFLAGS) +pzgetfile_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pzgetfile_SOURCES = \ + pzgetfile.c \ + pzgetfileConfig.c + dettool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) $(pxtools_CFLAGS) dettool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la Index: /trunk/ippTools/src/pzgetfile.c =================================================================== --- /trunk/ippTools/src/pzgetfile.c (revision 10028) +++ /trunk/ippTools/src/pzgetfile.c (revision 10028) @@ -0,0 +1,509 @@ +/* + * pzgetfile.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pxtools.h" +#include "pzgetfile.h" + +#define FILESET_LS_CMD "dsfilesetls" + +static bool go (pxConfig *config); +static psArray *parseFiles(pxConfig *config, const char *str); + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pzgetfileConfig(NULL, argc, argv); + + if (!go(config)) { + goto FAIL; + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_FAILURE); +} + +static bool go(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + bool status = false; + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + + psString filesetid = psMetadataLookupStr(&status, config->args, "-filesetid"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filesetid"); + return false; + } + if (!filesetid) { + psError(PS_ERR_UNKNOWN, true, "-filesetid is required"); + return false; + } + + psString camera = psMetadataLookupStr(&status, config->args, "-inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst"); + return NULL; + } + if (!camera) { + psError(PS_ERR_UNKNOWN, true, "-inst is required"); + return NULL; + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope"); + return NULL; + } + if (!telescope) { + psError(PS_ERR_UNKNOWN, true, "-telescope is required"); + return NULL; + } + + // invoke dsfilesetls + psString cmd = NULL; + psStringAppend(&cmd, "%s --uri %s", FILESET_LS_CMD, uri); + + psTrace("pzgetimfiles", PS_LOG_INFO, "cmd is: %s\n", cmd); + + FILE *output = popen(cmd, "r"); + psFree(cmd); + + if (!output) { + psError(PS_ERR_UNKNOWN, true, "popen() failed"); + return false; + } + + psString cmdOutput = psSlurpFile(output); + int exitStatus = pclose(output); + + if (exitStatus != 0) { + psError(PS_ERR_UNKNOWN, true, "%s failed with exit status %d", + FILESET_LS_CMD, exitStatus); + return false; + } + + // prase output of dsfilesetls + psArray *newImfiles = parseFiles(config, cmdOutput); + if (!newImfiles) { + // XXX not nessicarily an error + psError(PS_ERR_UNKNOWN, true, "no new files/imfiles"); + psFree(cmdOutput); + return false; + } + psFree(cmdOutput); + + // save the number of new Imfiles; + long imfiles = psArrayLength(newImfiles); + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // if the fileset was empty (no files) then we can bail out early + if (imfiles == 0) { + psFree(newImfiles); + + char *query = + "UPDATE summitExp" + " SET imfiles = %d" + " WHERE exp_id = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, imfiles, filesetid, camera, telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // remove the pzPendingExp entry for this exp (fileset) + { + char *query = + "DELETE FROM pzPendingExp" + " WHERE" + " exp_id = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, filesetid, camera, telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; + } + + // try not to insert duplicate pzPendingExp/pzPendingImfile entries + + // create a temp table + { + char *query = + "CREATE TEMPORARY TABLE incoming (exp_id VARCHAR(64), camera VARCHAR(64), telescope VARCHAR(64), bytes INT, md5sum VARCHAR(32), class VARCHAR(64), class_id VARCHAR(64), uri VARCHAR(255), PRIMARY KEY(exp_id, camera, telescope, class, class_id)) ENGINE=MEMORY"; + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newImfiles); + return false; + } + } + + // load the imfiles (files) into the temp table + { + char *query = "INSERT INTO incoming (exp_id, camera, telescope, bytes, md5sum, class, class_id, uri) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"; + + long inserted = p_psDBRunQueryPrepared(config->dbh, newImfiles, query); + if (inserted < 0) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newImfiles); + return false; + } + // sanity check that we actually inserted something + if (inserted == 0) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error -- we should have inserted at least one row"); + psFree(newImfiles); + return false; + } + } + + psFree(newImfiles); + + // queue the imfiles (files) in pzPendingImfile + { + char *query = + "INSERT INTO pzPendingImfile" + " SElECT" + " incoming.exp_id," + " incoming.camera," + " incoming.telescope," + " incoming.class," + " incoming.class_id," + " pzPendingExp.exp_tag" + " FROM incoming" + " JOIN pzPendingExp" + " USING(exp_id, camera, telescope)" + " LEFT JOIN summitImfile" + " USING(exp_id, camera, telescope, class, class_id)" + " WHERE" + " summitImfile.exp_id is NULL" + " AND summitImfile.camera is NULL" + " AND summitImfile.telescope is NULL" + " AND summitImfile.class is NULL" + " AND summitImfile.class_id is NULL"; + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // move imfiles (files) from the temp table into summitImfiles + { + char *query = + "INSERT INTO summitImfile" + " SElECT" + " incoming.exp_id," + " incoming.camera," + " incoming.telescope," + " incoming.bytes," + " incoming.md5sum," + " incoming.class," + " incoming.class_id," + " incoming.uri" + " FROM incoming" + " LEFT JOIN summitImfile" + " USING(exp_id, camera, telescope, class, class_id)" + " WHERE" + " summitImfile.exp_id is NULL" + " AND summitImfile.camera is NULL" + " AND summitImfile.telescope is NULL" + " AND summitImfile.class is NULL" + " AND summitImfile.class_id is NULL"; + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // update summitExp.imfiles (should have been NULL) + { + char *query = + "UPDATE summitExp" + " SET imfiles = (SELECT COUNT(*) FROM summitImfile" + " WHERE" + " exp_id = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'" + ")" + " WHERE" + " exp_id = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, filesetid, camera, telescope, filesetid, camera, telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // remove the pzPendingExp entry for this exp (fileset) + { + char *query = + "DELETE FROM pzPendingExp" + " WHERE" + " exp_id = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, filesetid, camera, telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static psArray *parseFiles(pxConfig *config, const char *str) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(str, NULL); + + // these are constants for all records parsed -- look them up before we do + // any work + bool status = false; + psString exp_id = psMetadataLookupStr(&status, config->args, "-filesetid"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for '-filesetid'"); + return NULL; + } + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "-filesetid is required"); + return NULL; + } + + psString camera = psMetadataLookupStr(&status, config->args, "-inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst"); + return NULL; + } + if (!camera) { + psError(PS_ERR_UNKNOWN, true, "-inst is required"); + return NULL; + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope"); + return NULL; + } + if (!telescope) { + psError(PS_ERR_UNKNOWN, true, "-telescope is required"); + return NULL; + } + + // split the string into lines + psList *doc = psStringSplit(str, "\n", false); + + psListIterator *lineCursor = psListIteratorAlloc(doc, 0, false); + + psArray *pzPendingImfiles = psArrayAllocEmpty(psListLength(doc)); + psString line; + while ((line = psListGetAndIncrement(lineCursor))) { + psTrace("pzgetimfiles", PS_LOG_INFO, "parsing line: %s\n", line); + + // split line into tokens + psList *tokens = psStringSplit(line, " ", false); + + // check to see if this line is a comment (or if the first token is + // NULL) + if (!psListGet(tokens, 0) || *((char *)psListGet(tokens, 0)) == '#') { + psFree(tokens); + continue; + } + + // check that we have the right number of tokens + // print "# uri fileid bytes md5sum type \n"; + if (psListLength(tokens) != 5) { + psError(PS_ERR_UNKNOWN, true, "invalid line format: %s", line); + return false; + } + + // find the values of interest + psListIterator *tokenCursor = psListIteratorAlloc(tokens, 0, false); + char *uri = psListGetAndIncrement(tokenCursor); + char *class_id = psListGetAndIncrement(tokenCursor); // fileid + char *bytes = psListGetAndIncrement(tokenCursor); // bytes + char *md5sum = psListGetAndIncrement(tokenCursor); // md5sum + char *class = psListGetAndIncrement(tokenCursor); // type + + // create a new metadata to represent this line and it's values + psMetadata *md = psMetadataAlloc(); + if (!psMetadataAddStr(md, PS_LIST_TAIL, "exp_id", 0, NULL, exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "camera", 0, NULL, camera)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "telescope", 0, NULL, telescope)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item telescope"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddS32(md, PS_LIST_TAIL, "bytes", 0, NULL, (psS32)atoi(bytes))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item bytes"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "md5sum", 0, NULL, md5sum)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item md5sum"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "class", 0, NULL, class)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "class_id", 0, NULL, class_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class_id"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "uri", 0, NULL, uri)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item uri"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + + // must be freed after the new metadata is built -- holds the strings + psFree(tokenCursor); + psFree(tokens); + + // debugging + if (psTraceGetLevel("pzgetimfiles") == PS_LOG_INFO) { + psString doc = psMetadataConfigFormat(md); + psTrace("pzgetimfiles", PS_LOG_INFO, "parsed line as:\n %s\n", doc); + psFree(doc); + } + + psArrayAdd(pzPendingImfiles, 0, md); + + psFree(md); + } + + psFree(lineCursor); + psFree(doc); + + return pzPendingImfiles;; +} Index: /trunk/ippTools/src/pzgetfile.h =================================================================== --- /trunk/ippTools/src/pzgetfile.h (revision 10028) +++ /trunk/ippTools/src/pzgetfile.h (revision 10028) @@ -0,0 +1,27 @@ +/* + * pzgetfile.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PZGETFILE_H +#define PZGETFILE 1 + +#include "pxtools.h" + +pxConfig *pzgetfileConfig(pxConfig *config, int argc, char **argv); + +#endif // PZGETFILE_H Index: /trunk/ippTools/src/pzgetfileConfig.c =================================================================== --- /trunk/ippTools/src/pzgetfileConfig.c (revision 10028) +++ /trunk/ippTools/src/pzgetfileConfig.c (revision 10028) @@ -0,0 +1,81 @@ +/* + * pzgetfileConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pxtools.h" + +pxConfig *pzgetfileConfig(pxConfig *config, int argc, char **argv) { + if (!config) { + config = pxConfigAlloc(); + } + + config->modules = pmConfigRead(&argc, argv); + + if (! config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); + exit(EXIT_FAILURE); + } + + psMetadata *args = psMetadataAlloc(); + psMetadataAddStr(args , PS_LIST_TAIL, "-uri", 0, + "DataStore FileSet URI", ""); + psMetadataAddStr(args , PS_LIST_TAIL, "-filesetid", 0, + "FileSet ID", ""); + psMetadataAddStr(args, PS_LIST_TAIL, "-inst", 0, + "search by camera", NULL); + psMetadataAddStr(args, PS_LIST_TAIL, "-telescope", 0, + "search by telescope", NULL); + + + bool status = false; + if (!psArgumentParse(args, &argc, argv) + || argc != 1 + || strcmp(psMetadataLookupStr(&status, args, "-uri"), "") == 0 + || strcmp(psMetadataLookupStr(&status, args, "-filesetid"), "") == 0 + ) { + fprintf(stderr, "error parsing arguments\n"); + printf("\nPan-STARRS Phase Z Search Tool\n"); + printf("Usage: %s -uri -filesetid \n\n", argv[0]); + psArgumentHelp(args); + psFree(config); + exit(EXIT_FAILURE); + } + + config->args = args; + // don't free args here as it's silly to increment the ref count then + // "free" it + + // define Database handle, if used + config->dbh = pmConfigDB(config->modules); + if(!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't connect to db\n"); + psFree(config); + exit(EXIT_FAILURE); + } + + return config; +}