Index: trunk/ippTools/src/Makefile.am =================================================================== --- trunk/ippTools/src/Makefile.am (revision 14589) +++ trunk/ippTools/src/Makefile.am (revision 14600) @@ -14,4 +14,5 @@ warptool \ pzgetimfiles \ + ttree \ pztool @@ -65,4 +66,10 @@ # for pxtools.h AM_CPPFLAGS = -I$(top_srcdir)/src$ + + +ttree_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +ttree_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +ttree_SOURCES = \ + ttree.c pztool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) Index: trunk/ippTools/src/pzgetexpConfig.c =================================================================== --- trunk/ippTools/src/pzgetexpConfig.c (revision 14589) +++ trunk/ippTools/src/pzgetexpConfig.c (revision 14600) @@ -46,9 +46,9 @@ psMetadata *args = psMetadataAlloc(); psMetadataAddStr(args , PS_LIST_TAIL, "-uri", 0, - "DataStore product URI", ""); + "DataStore product URI (required)", ""); psMetadataAddStr(args , PS_LIST_TAIL, "-inst", 0, - "camera name", ""); + "camera name (required)", ""); psMetadataAddStr(args , PS_LIST_TAIL, "-telescope", 0, - "telescope name", ""); + "telescope name (required)", ""); bool status = false; @@ -60,5 +60,5 @@ ) { fprintf(stderr, "error parsing arguments\n"); - printf("\nPan-STARRS Phase Z Search Tool\n"); + printf("\nPan-STARRS Phase Z Get Exposures Tool\n"); printf("Usage: %s -uri -inst -telescope \n\n", argv[0]); Index: trunk/ippTools/src/pzgetimfiles.c =================================================================== --- trunk/ippTools/src/pzgetimfiles.c (revision 14589) +++ trunk/ippTools/src/pzgetimfiles.c (revision 14600) @@ -274,6 +274,5 @@ " incoming.telescope," " incoming.class," - " incoming.class_id," - " NULL" // exp_id, will be assigned later + " incoming.class_id" " FROM incoming" " JOIN pzPendingExp" @@ -367,79 +366,4 @@ } - // copy the summitExp row into newExp in order to create a new exp_id - { - psString exp_id = pxGenExpTag(config, filesetid); - if (!exp_id) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - char *query = - "INSERT INTO newExp" - " SElECT" - " '%s'," // exp_id - " summitExp.exp_name," - " summitExp.camera," - " summitExp.telescope," - " summitExp.dateobs," - " summitExp.exp_type," - " summitExp.imfiles," - " 0" // error flags - " FROM summitExp" - " WHERE" - " summitExp.exp_name = '%s'" - " AND summitExp.camera = '%s'" - " AND summitExp.telescope = '%s'"; - - if (!p_psDBRunQuery(config->dbh, query, exp_id, filesetid, camera, telescope)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(exp_id); - return false; - } - psFree(exp_id); - - // sanity check: we should have inserted only one row - psU64 affected = psDBAffectedRows(config->dbh); - if (psDBAffectedRows(config->dbh) != 1) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "should have affected 1 row but %" PRIu64 " rows were modified", affected); - return false; - } - } - - // set the exp_id for the imfiles (files) in the exposure (fileset) that - // we are investigating - { - char *query = - "UPDATE pzPendingImfile" - " SET exp_id = (SELECT exp_id from newExp" - " WHERE exp_name = '%s'" - " AND camera = '%s'" - " AND telescope = '%s')" - " WHERE" - " exp_name = '%s'" - " AND camera = '%s'" - " AND telescope = '%s'"; - if (!p_psDBRunQuery(config->dbh, query, filesetid, camera, telescope, filesetid, camera, telescope)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - } - // cp the pzPendingExp entry to pzDoneExp { Index: trunk/ippTools/src/pzgetimfilesConfig.c =================================================================== --- trunk/ippTools/src/pzgetimfilesConfig.c (revision 14589) +++ trunk/ippTools/src/pzgetimfilesConfig.c (revision 14600) @@ -46,11 +46,11 @@ psMetadata *args = psMetadataAlloc(); psMetadataAddStr(args , PS_LIST_TAIL, "-uri", 0, - "DataStore FileSet URI", ""); + "DataStore FileSet URI (required)", NULL); psMetadataAddStr(args , PS_LIST_TAIL, "-filesetid", 0, - "FileSet ID", ""); + "FileSet ID (required)", NULL); psMetadataAddStr(args, PS_LIST_TAIL, "-inst", 0, - "search by camera", NULL); + "Camera ID (required)", NULL); psMetadataAddStr(args, PS_LIST_TAIL, "-telescope", 0, - "search by telescope", NULL); + "Telescope ID (required)", NULL); @@ -58,9 +58,11 @@ if (!psArgumentParse(args, &argc, argv) || argc != 1 - || strcmp(psMetadataLookupStr(&status, args, "-uri"), "") == 0 - || strcmp(psMetadataLookupStr(&status, args, "-filesetid"), "") == 0 + || (psMetadataLookupStr(&status, args, "-uri") == NULL) + || (psMetadataLookupStr(&status, args, "-filesetid") == NULL) + || (psMetadataLookupStr(&status, args, "-inst") == NULL) + || (psMetadataLookupStr(&status, args, "-telescope") == NULL) ) { fprintf(stderr, "error parsing arguments\n"); - printf("\nPan-STARRS Phase Z Search Tool\n"); + printf("\nPan-STARRS Phase Z Get Imfiles Tool\n"); printf("Usage: %s -uri -filesetid \n\n", argv[0]); psArgumentHelp(args); Index: trunk/ippTools/src/pztool.c =================================================================== --- trunk/ippTools/src/pztool.c (revision 14589) +++ trunk/ippTools/src/pztool.c (revision 14600) @@ -35,4 +35,6 @@ static bool copydoneMode(pxConfig *config); +static bool copydoneCompleteExp(pxConfig *config); + # define MODECASE(caseName, func) \ case caseName: \ @@ -196,6 +198,5 @@ psString query = psStringCopy( "SELECT" - " summitImfile.*," - " pzPendingImfile.exp_id" + " summitImfile.*" " FROM pzPendingImfile" " JOIN summitImfile" @@ -324,42 +325,4 @@ } - // insert new imfile into newImfile - { - char *query = - "INSERT INTO newImfile" - " SElECT" - " pzPendingImfile.exp_id," - " pzPendingImfile.class," - " pzPendingImfile.class_id," - " '%s'," // uri of downloaded file - " 0" // error flags - " FROM pzPendingImfile" - " WHERE" - " pzPendingImfile.exp_name = '%s'" - " AND pzPendingImfile.camera = '%s'" - " AND pzPendingImfile.telescope = '%s'" - " AND pzPendingImfile.class = '%s'" - " AND pzPendingImfile.class_id = '%s'"; - - if (!p_psDBRunQuery(config->dbh, query, uri, exp_name, camera, telescope, class, class_id)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - // sanity check: we should have inserted only one row - psU64 affected = psDBAffectedRows(config->dbh); - if (psDBAffectedRows(config->dbh) != 1) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "should have affected 1 row but %" PRIu64 " rows were modified", affected); - return false; - } - } // cp the imfile into pzDoneImfile @@ -441,4 +404,127 @@ } + if (!copydoneCompleteExp(config)) { + psError(PS_ERR_UNKNOWN, false, "copydoneCompleteExp() failed"); + return false; + } + return true; } + +static bool copydoneCompleteExp(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // find all exposures that have had all of their imfiles downloaded but do + // not appear in newExp + psString query = pxDataGet("pztool_find_completed_exp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pztool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so all newExp's can start off with a state of run + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + pzDoneExpRow *doneExp = pzDoneExpObjectFromMetadata(row); + + if (!newExpInsert(config->dbh, + 0x0, // exp_id + doneExp->exp_name, // tmp_exp_name + doneExp->camera, // tmp_camera + doneExp->telescope, // tmp_telescope + "run", // state + NULL, // workdir + "dirty", // workdir state + NULL // reduction class + ) + ) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(doneExp); + psFree(output); + return false; + } + + psS64 exp_id = psDBLastInsertID(config->dbh); + + // insert new pzDoneImfile into newImfile + { + char *query = + "INSERT INTO newImfile" + " SElECT" + " %" PRId64 "," // exp_id + " pzDoneImfile.class_id," // tmp_class_id + " pzDoneImfile.uri" // uri + " FROM pzDoneImfile" + " WHERE" + " pzDoneImfile.exp_name = '%s'" + " AND pzDoneImfile.camera = '%s'" + " AND pzDoneImfile.telescope = '%s'"; + + if (!p_psDBRunQuery(config->dbh, query, exp_id, doneExp->exp_name, doneExp->camera, doneExp->telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(doneExp); + psFree(output); + return false; + } + + // sanity check: we should have inserted only one row + psU64 affected = psDBAffectedRows(config->dbh); + if (psDBAffectedRows(config->dbh) != 1) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row but %" PRIu64 " rows were modified", affected); + psFree(doneExp); + psFree(output); + return false; + } + } + + psFree(doneExp); + } + + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} Index: trunk/ippTools/src/pztoolConfig.c =================================================================== --- trunk/ippTools/src/pztoolConfig.c (revision 14589) +++ trunk/ippTools/src/pztoolConfig.c (revision 14600) @@ -45,5 +45,5 @@ // -seen psMetadata *seenArgs = psMetadataAlloc(); - psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-exp_id", 0, + psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-inst", 0, @@ -58,5 +58,5 @@ // -pendingexp psMetadata *pendingexpArgs = psMetadataAlloc(); - psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-exp_id", 0, + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-inst", 0, @@ -71,5 +71,5 @@ // -pendingimfile psMetadata *pendingimfileArgs = psMetadataAlloc(); - psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-exp_id", 0, + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-inst", 0, @@ -117,37 +117,15 @@ psMetadata *argSets = psMetadataAlloc(); - // find which mode we're running under - PXTOOL_MODE("-seen", PZTOOL_MODE_SEEN, seenArgs); - PXTOOL_MODE("-pendingexp", PZTOOL_MODE_PENDINGEXP, pendingexpArgs); - PXTOOL_MODE("-pendingimfile", PZTOOL_MODE_PENDINGIMFILE,pendingimfileArgs); - PXTOOL_MODE("-copydone", PZTOOL_MODE_COPYDONE, copydoneArgs); + psMetadata *modes = psMetadataAlloc(); - bool argErr = false; - if (config->mode == PZTOOL_MODE_NONE) { - argErr = true; - fprintf (stderr, "mode argument is required\n"); - } else if (! psArgumentParse(config->args, &argc, argv) || argc != 1) { - argErr = true; - fprintf (stderr, "error parsing arguments\n"); - } + PXTOOL_ADD_MODE("-seen", "", PZTOOL_MODE_SEEN, seenArgs); + PXTOOL_ADD_MODE("-pendingexp", "", PZTOOL_MODE_PENDINGEXP, pendingexpArgs); + PXTOOL_ADD_MODE("-pendingimfile", "", PZTOOL_MODE_PENDINGIMFILE,pendingimfileArgs); + PXTOOL_ADD_MODE("-copydone", "", PZTOOL_MODE_COPYDONE, copydoneArgs); - if (argErr) { - printf("\nPan-STARRS Detrend Tool\n"); - printf("Usage: %s []\n\n", argv[0]); - printf(" :\n\n"); - - psMetadataIterator *iter = psMetadataIteratorAlloc(argSets, 0, NULL); - psMetadataItem *item = NULL; - while ((item = psMetadataGetAndIncrement(iter))) { - if (!item->type == PS_DATA_METADATA) { - psAbort("all options must be specified as a metadata"); - } - - fprintf(stdout, "%s ", item->name); - psArgumentHelp(item->data.md); - } - psFree(iter); - + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); psFree(argSets); + psFree(modes); psFree(config); return NULL; @@ -155,4 +133,5 @@ psFree(argSets); + psFree(modes); // setup search criterion @@ -173,5 +152,5 @@ config->where = psMetadataAlloc(); - addWhereStr(exp_id); + addWhereStr(exp_name); addWhereStr(telescope); // convert '-inst' to 'camera' @@ -206,8 +185,4 @@ } - // save argv/argc - config->argv = argv; - config->argc = argc; - return config; }