Index: /branches/eam_branch_20080719/ippTools/share/chiptool_processedimfile.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/chiptool_processedimfile.sql (revision 19002) +++ /branches/eam_branch_20080719/ippTools/share/chiptool_processedimfile.sql (revision 19003) @@ -6,4 +6,5 @@ chipProcessedImfile.bg_mean_stdev, chipProcessedImfile.path_base, + chipRun.state, rawExp.exp_id, rawExp.exp_tag, Index: /branches/eam_branch_20080719/ippTools/share/difftool_skyfile.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/difftool_skyfile.sql (revision 19002) +++ /branches/eam_branch_20080719/ippTools/share/difftool_skyfile.sql (revision 19003) @@ -2,9 +2,28 @@ diffRun.skycell_id, diffRun.tess_id, - diffSkyfile.* + diffRun.state, + diffSkyfile.*, + rawExp.exp_id, + rawExp.camera FROM diffRun JOIN diffSkyfile USING(diff_id) -WHERE - diffRun.state = 'new' - +-- WS: my new stuff begins here +JOIN diffInputSkyfile + ON diffInputSkyfile.diff_id = diffRun.diff_id + AND diffInputSkyfile.template = 0 +JOIN warpRun + ON diffInputSkyfile.warp_id = warpRun.warp_id +JOIN fakeRun + ON warpRun.fake_id = fakeRun.fake_id +JOIN camRun + ON camRun.cam_id = fakeRun.cam_id +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id +-- WS: my new stuff ends here + WHERE + -- bogus test to allow appending conditionals + -- XXX EAM : this is not needed if we modify the difftool.c code : fix + diffRun.diff_id IS NOT NULL Index: /branches/eam_branch_20080719/ippTools/share/stacktool_sumskyfile.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/stacktool_sumskyfile.sql (revision 19002) +++ /branches/eam_branch_20080719/ippTools/share/stacktool_sumskyfile.sql (revision 19003) @@ -1,7 +1,23 @@ SELECT - stackSumSkyfile.* + stackSumSkyfile.*, + stackRun.state, + rawExp.camera FROM stackRun JOIN stackSumSkyfile USING(stack_id) +JOIN warpRun + -- just need 1 warp to find the camera + ON warpRun.warp_id = (SELECT warp_id FROM stackInputSkyfile + WHERE stackInputSkyfile.stack_id = stackRun.stack_id limit 1) +JOIN fakeRun + ON warpRun.fake_id = fakeRun.fake_id +JOIN camRun + ON camRun.cam_id = fakeRun.cam_id +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id WHERE - stackRun.state = 'new' +-- bogus test; just here so there there is a 'WHERE' stmt to append conditionals too +-- XXX EAM : this is not needed : fix the stacktool code. + stackRun.stack_id is NOT NULL Index: /branches/eam_branch_20080719/ippTools/share/warptool_warped.sql =================================================================== --- /branches/eam_branch_20080719/ippTools/share/warptool_warped.sql (revision 19002) +++ /branches/eam_branch_20080719/ippTools/share/warptool_warped.sql (revision 19003) @@ -1,7 +1,20 @@ SELECT - warpSkyfile.* + warpSkyfile.*, + warpRun.state, + rawExp.exp_id, + rawExp.camera FROM warpRun JOIN warpSkyfile USING(warp_id) +JOIN fakeRun + ON warpRun.fake_id = fakeRun.fake_id +JOIN camRun + ON camRun.cam_id = fakeRun.cam_id +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id WHERE - warpRun.state = 'new' +-- bogus test; just here so there there is a 'WHERE' stmt to append conditionals too +-- XXX EAM : not needed: fix warptool.c + warpRun.warp_id is NOT NULL Index: /branches/eam_branch_20080719/ippTools/src/chiptool.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/chiptool.c (revision 19002) +++ /branches/eam_branch_20080719/ippTools/src/chiptool.c (revision 19003) @@ -523,4 +523,5 @@ PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "=="); PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); Index: /branches/eam_branch_20080719/ippTools/src/chiptoolConfig.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/chiptoolConfig.c (revision 19002) +++ /branches/eam_branch_20080719/ippTools/src/chiptoolConfig.c (revision 19003) @@ -192,12 +192,13 @@ // -processedimfile psMetadata *processedimfileArgs = psMetadataAlloc(); - psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "define chip ID", 0); - psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0); - psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID", NULL); + psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "define chip ID", 0); + psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure name", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID", NULL); psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-inst", 0, "define camera of interest", NULL); - psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-filter", 0, "define filter of interest", NULL); - psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); - psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-filter", 0, "define filter of interest", NULL); + psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -updateprocessedimfile Index: /branches/eam_branch_20080719/ippTools/src/difftool.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/difftool.c (revision 19002) +++ /branches/eam_branch_20080719/ippTools/src/difftool.c (revision 19003) @@ -525,7 +525,9 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); - PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffSkyfile.diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffSkyfile.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "diffSkyfile.tess_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -539,5 +541,5 @@ if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "diffSkyfile"); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); @@ -1125,5 +1127,4 @@ psFree(output); - - return true; -} + return true; +} Index: /branches/eam_branch_20080719/ippTools/src/difftoolConfig.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/difftoolConfig.c (revision 19002) +++ /branches/eam_branch_20080719/ippTools/src/difftoolConfig.c (revision 19003) @@ -104,4 +104,6 @@ psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID", NULL); psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID", NULL); + psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0); + psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-exp_name", 0, "define exposure name", NULL); psMetadataAddU64(diffskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); Index: /branches/eam_branch_20080719/ippTools/src/pstamptool.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/pstamptool.c (revision 19002) +++ /branches/eam_branch_20080719/ippTools/src/pstamptool.c (revision 19003) @@ -220,9 +220,9 @@ PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); // PXOPT_LOOKUP_STR(outFileset, config->args, "-out_fileset", true, false); - PXOPT_LOOKUP_STR(ds_id, config->args, "-ds_id", false, false); + PXOPT_LOOKUP_S32(ds_id, config->args, "-ds_id", false, false); char *query ="INSERT INTO pstampRequest" " (state, uri, ds_id, fault)" - " VALUES( 'new', '%s', %s, 0 )"; + " VALUES( 'new', '%s', %" PRId64 ", 0 )"; if (!p_psDBRunQuery(config->dbh, query, uri, ds_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: /branches/eam_branch_20080719/ippTools/src/pstamptoolConfig.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/pstamptoolConfig.c (revision 19002) +++ /branches/eam_branch_20080719/ippTools/src/pstamptoolConfig.c (revision 19003) @@ -64,5 +64,5 @@ psMetadata *addreqArgs = psMetadataAlloc(); psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0, "define request file uri", NULL); - psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-ds_id", 0, "define request ds_id", NULL); + psMetadataAddS32(addreqArgs, PS_LIST_TAIL, "-ds_id", 0, "define request ds_id", 0); // psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-out_fileset", 0, "define request output_fileset", NULL); Index: /branches/eam_branch_20080719/ippTools/src/pxadmin.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/pxadmin.c (revision 19002) +++ /branches/eam_branch_20080719/ippTools/src/pxadmin.c (revision 19003) @@ -122,9 +122,12 @@ { - bool status; char line[128], answer[128]; - // XXX use the values defined in config->dbh? - psString dbName = psMetadataLookupStr(&status, config->modules->complete, "DBNAME"); + psMetadataItem *name = pmConfigUserSite(config->modules, "DBNAME", PS_DATA_STRING); + if (!name) { + psError(PS_ERR_UNKNOWN, false, "Unable to determine database name."); + return false; + } + psString dbName = name->data.str; fprintf(stdout, "*** delete the tables from database %s? ***\n", dbName); @@ -150,5 +153,10 @@ sscanf(line, "%s", answer); - psString dbPassword = psMetadataLookupStr(&status, config->modules->complete, "DBPASSWORD"); + psMetadataItem *pass = pmConfigUserSite(config->modules, "DBPASSWORD", PS_DATA_STRING); + if (!pass) { + psError(PS_ERR_UNKNOWN, false, "Unable to determine database password."); + return false; + } + psString dbPassword = pass->data.str; if (strcmp (answer, dbPassword)) { psError(PS_ERR_UNKNOWN, true, "invalid passwd - tables NOT deleleted"); Index: /branches/eam_branch_20080719/ippTools/src/pxadminConfig.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/pxadminConfig.c (revision 19002) +++ /branches/eam_branch_20080719/ippTools/src/pxadminConfig.c (revision 19003) @@ -27,20 +27,27 @@ #include "pxadmin.h" -static void pxAdminShowDB (pxConfig *config, const char *program) { - - bool status; - +static void pxAdminShowDB (const pxConfig *config, const char *program) +{ fprintf (stderr, "\nPan-STARRS DataBase Admin Tool\n\n"); fprintf (stderr, "Usage: %s [mode]\n", program); fprintf (stderr, " [mode] : -create | -delete\n\n"); - // XXX EAM : this should be done with psDB function calls - psString dbServer = psMetadataLookupStr(&status, config->modules->complete, "DBSERVER"); - psString dbUsername = psMetadataLookupStr(&status, config->modules->complete, "DBUSER"); - psString dbName = psMetadataLookupStr(&status, config->modules->complete, "DBNAME"); - psS32 dbPort = psMetadataLookupS32(&status, config->modules->complete, "DBPORT"); + psMetadataItem *server = pmConfigUserSite(config->modules, "DBSERVER", PS_DATA_STRING); + psMetadataItem *user = pmConfigUserSite(config->modules, "DBUSER", PS_DATA_STRING); + psMetadataItem *name = pmConfigUserSite(config->modules, "DBNAME", PS_DATA_STRING); + psMetadataItem *port = pmConfigUserSite(config->modules, "DBPORT", PS_TYPE_S32); - fprintf (stderr, "connecting to %s as %s (port %d)\n", dbServer, dbUsername, dbPort); - fprintf (stderr, "using database %s\n\n", dbName); + if (!server || !user || !name) { + psErrorClear(); + psWarning("Unable to determine database connection details."); + return; + } + if (!port) { + psErrorClear(); + } + + fprintf (stderr, "connecting to %s as %s (port %d)\n", server->data.str, user->data.str, + port ? port->data.S32 : 0); + fprintf (stderr, "using database %s\n\n", name->data.str); return; Index: /branches/eam_branch_20080719/ippTools/src/pzgetexp.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/pzgetexp.c (revision 19002) +++ /branches/eam_branch_20080719/ippTools/src/pzgetexp.c (revision 19003) @@ -149,5 +149,21 @@ if (!psDBTransaction(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newSummitExps); return false; + } + + // increase memory table size limits; deafult is 16MB + { + // 512MB + char *query = "SET max_heap_table_size = 1024*1024*512"; + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newSummitExps); + return false; + } } Index: /branches/eam_branch_20080719/ippTools/src/stacktool.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/stacktool.c (revision 19002) +++ /branches/eam_branch_20080719/ippTools/src/stacktool.c (revision 19003) @@ -358,9 +358,8 @@ if (!stackRunPrintObjects(stdout, list, !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print object"); - return false; - } - + psFree(list); + return false; + } psFree(list); - return true; } @@ -714,6 +713,8 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); - PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -727,5 +728,5 @@ if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "stackSumSkyfile"); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); Index: /branches/eam_branch_20080719/ippTools/src/stacktoolConfig.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/stacktoolConfig.c (revision 19002) +++ /branches/eam_branch_20080719/ippTools/src/stacktoolConfig.c (revision 19003) @@ -126,4 +126,6 @@ psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", NULL); psMetadataAddU64(sumskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(sumskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -165,5 +167,5 @@ PXOPT_ADD_MODE("-tosum", "", STACKTOOL_MODE_TOSUM, tosumArgs); PXOPT_ADD_MODE("-addsumskyfile", "", STACKTOOL_MODE_ADDSUMSKYFILE, addsumskyfileArgs); - PXOPT_ADD_MODE("-sumskyfile", "", STACKTOOL_MODE_SUMSKYFILE, sumskyfileArgs); + PXOPT_ADD_MODE("-sumskyfile", "list results of stackRun", STACKTOOL_MODE_SUMSKYFILE, sumskyfileArgs); PXOPT_ADD_MODE("-revertsumskyfile","", STACKTOOL_MODE_REVERTSUMSKYFILE, revertsumskyfileArgs); PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", STACKTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); Index: /branches/eam_branch_20080719/ippTools/src/warptool.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/warptool.c (revision 19002) +++ /branches/eam_branch_20080719/ippTools/src/warptool.c (revision 19003) @@ -916,9 +916,10 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); - PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyfile.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyfile.tess_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -933,5 +934,5 @@ if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "warpSkyfile"); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); Index: /branches/eam_branch_20080719/ippTools/src/warptoolConfig.c =================================================================== --- /branches/eam_branch_20080719/ippTools/src/warptoolConfig.c (revision 19002) +++ /branches/eam_branch_20080719/ippTools/src/warptoolConfig.c (revision 19003) @@ -182,4 +182,5 @@ psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID", NULL); psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure tag", 0); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure tag", 0); psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-fake_id", 0, "define phase 3 version of exposure tag", 0); psMetadataAddU64(warpedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);