Index: /trunk/ippTools/share/Makefile.am =================================================================== --- /trunk/ippTools/share/Makefile.am (revision 26926) +++ /trunk/ippTools/share/Makefile.am (revision 26927) @@ -100,4 +100,6 @@ difftool_definewarpwarp_select.sql \ difftool_definewarpwarp_insert.sql \ + difftool_definestackstack_part0.sql \ + difftool_definestackstack_part1.sql \ difftool_donecleanup.sql \ difftool_export_input_skyfile.sql \ Index: /trunk/ippTools/share/difftool_definestackstack_part0.sql =================================================================== --- /trunk/ippTools/share/difftool_definestackstack_part0.sql (revision 26927) +++ /trunk/ippTools/share/difftool_definestackstack_part0.sql (revision 26927) @@ -0,0 +1,41 @@ +SELECT DISTINCT + stackRun.data_group AS INPUT_data_group, + stackRun.dist_group AS INPUT_dist_group, + stackRun.filter AS INPUT_filter, + stackRun.label AS INPUT_label, + stackRun.tess_id AS INPUT_tess_id +FROM stackRun JOIN stackSumSkyfile USING(stack_id) +JOIN ( + SELECT + MAX(stackRun.stack_id) as max_stack_id, + stackRun.filter, + stackRun.label as template_label, + stackRun.tess_id, + stackRun.skycell_id, + stackSumSkyfile.good_frac + FROM stackRun JOIN stackSumSkyfile USING(stack_id) + WHERE + stackRun.state = 'full' AND + stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0 + -- template where hook %s + GROUP BY + skycell_id, + filter +) AS template ON stackRun.filter = template.filter + AND stackRun.tess_id = template.tess_id + AND stackRun.skycell_id = template.skycell_id +LEFT JOIN ( + SELECT + diffRun.diff_id, + stackRun.stack_id + FROM diffRun JOIN diffInputSkyfile USING(diff_id) + JOIN stackRun ON stackRun.stack_id = diffInputSkyfile.stack1 + JOIN stackSumSkyfile USING(stack_id) + WHERE 1 + -- diff where hook %s + -- input where hook %s +) AS diffExp ON stackRun.stack_id = diffExp.stack_id +WHERE + stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0 + -- input where hook again %s +ORDER BY stackRun.data_group,stackRun.filter Index: /trunk/ippTools/share/difftool_definestackstack_part1.sql =================================================================== --- /trunk/ippTools/share/difftool_definestackstack_part1.sql (revision 26927) +++ /trunk/ippTools/share/difftool_definestackstack_part1.sql (revision 26927) @@ -0,0 +1,46 @@ +SELECT + stackRun.stack_id, + stackRun.data_group, + stackRun.filter, + stackRun.label, + stackRun.tess_id, + stackRun.skycell_id, + stackSumSkyfile.good_frac, + template.max_stack_id, + template.template_label, + diffExp.diff_id +FROM stackRun JOIN stackSumSkyfile USING(stack_id) +JOIN ( + SELECT + MAX(stackRun.stack_id) as max_stack_id, + stackRun.filter, + stackRun.label as template_label, + stackRun.tess_id, + stackRun.skycell_id, + stackSumSkyfile.good_frac + FROM stackRun JOIN stackSumSkyfile USING(stack_id) + WHERE + stackRun.state = 'full' AND + stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0 + -- template where hook %s + GROUP BY + skycell_id, + filter +) AS template ON stackRun.filter = template.filter + AND stackRun.tess_id = template.tess_id + AND stackRun.skycell_id = template.skycell_id +LEFT JOIN ( + SELECT + diffRun.diff_id, + stackRun.stack_id + FROM diffRun JOIN diffInputSkyfile USING(diff_id) + JOIN stackRun ON stackRun.stack_id = diffInputSkyfile.stack1 + JOIN stackSumSkyfile USING(stack_id) + WHERE 1 + -- diff where hook %s + -- input where hook %s +) AS diffExp ON stackRun.stack_id = diffExp.stack_id +WHERE + stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0 + -- input where hook again %s +ORDER BY stackRun.data_group,stackRun.filter,stackRun.skycell_id Index: /trunk/ippTools/share/difftool_pendingcleanuprun.sql =================================================================== --- /trunk/ippTools/share/difftool_pendingcleanuprun.sql (revision 26926) +++ /trunk/ippTools/share/difftool_pendingcleanuprun.sql (revision 26927) @@ -25,4 +25,5 @@ USING(exp_id) WHERE + @INNERCONSTRAINT@ (diffRun.state = 'goto_cleaned' OR diffRun.state = 'goto_scrubbed' OR @@ -52,4 +53,5 @@ USING(exp_id) WHERE + @INNERCONSTRAINT@ (diffRun.state = 'goto_cleaned' OR diffRun.state = 'goto_scrubbed' OR @@ -64,6 +66,6 @@ JOIN diffInputSkyfile USING(diff_id) - JOIN stackSumSkyfile - ON diffInputSkyfile.stack1 = stackSumSkyfile.stack_id +-- JOIN stackSumSkyfile +-- ON diffInputSkyfile.stack1 = stackSumSkyfile.stack_id JOIN stackInputSkyfile ON diffInputSkyfile.stack1 = stackInputSkyfile.stack_id @@ -79,4 +81,5 @@ USING(exp_id) WHERE + @INNERCONSTRAINT@ (diffRun.state = 'goto_cleaned' OR diffRun.state = 'goto_scrubbed' OR @@ -91,6 +94,6 @@ JOIN diffInputSkyfile USING(diff_id) - JOIN stackSumSkyfile - ON diffInputSkyfile.stack2 = stackSumSkyfile.stack_id +-- JOIN stackSumSkyfile +-- ON diffInputSkyfile.stack2 = stackSumSkyfile.stack_id JOIN stackInputSkyfile ON diffInputSkyfile.stack2 = stackInputSkyfile.stack_id @@ -106,4 +109,5 @@ USING(exp_id) WHERE + @INNERCONSTRAINT@ (diffRun.state = 'goto_cleaned' OR diffRun.state = 'goto_scrubbed' OR Index: /trunk/ippTools/src/difftool.c =================================================================== --- /trunk/ippTools/src/difftool.c (revision 26926) +++ /trunk/ippTools/src/difftool.c (revision 26927) @@ -43,4 +43,5 @@ static bool definewarpstackMode(pxConfig *config); static bool definewarpwarpMode(pxConfig *config); +static bool definestackstackMode(pxConfig *config); static bool pendingcleanuprunMode(pxConfig *config); static bool pendingcleanupskyfileMode(pxConfig *config); @@ -89,4 +90,5 @@ MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackMode); MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP, definewarpwarpMode); + MODECASE(DIFFTOOL_MODE_DEFINESTACKSTACK, definestackstackMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode); @@ -1831,4 +1833,262 @@ } +static bool definestackstackMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *stack1Where = psMetadataAlloc(); + psMetadata *stack2Where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, stack1Where, "-tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, stack2Where, "-tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, stack1Where, "-filter", "stackRun.filter", "=="); + PXOPT_COPY_STR(config->args, stack2Where, "-filter", "stackRun.filter", "=="); + PXOPT_COPY_STR(config->args, stack1Where, "-skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, stack2Where, "-skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, stack1Where, "-input_label", "stackRun.label","=="); + PXOPT_COPY_STR(config->args, stack2Where, "-template_label", "stackRun.label","=="); + PXOPT_COPY_F32(config->args, stack1Where, "-good_frac", "stackSumSkyfile.good_frac", ">="); + PXOPT_COPY_F32(config->args, stack2Where, "-good_frac", "stackSumSkyfile.good_frac", ">="); + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); // required option + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); // option + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); // option + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(newTemplates,config->args,"-new-templates", false); + + PXOPT_LOOKUP_BOOL(available, config->args, "-available", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + if (!(label)) { + PXOPT_LOOKUP_STR(label,config->args, "-input_label",true,false); + } + + // Organize the config information into queries. + psString stack1Query = NULL; + psString stack2Query = NULL; + + if (psListLength(stack1Where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(stack1Where, NULL); + psStringAppend(&stack1Query, "\n AND %s", whereClause); + psFree(whereClause); + } else { + stack1Query = psStringCopy("\n"); + } + psFree(stack1Where); + + if (psListLength(stack2Where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(stack2Where, NULL); + psStringAppend(&stack2Query, "\n AND %s", whereClause); + psFree(whereClause); + } else { + stack2Query = psStringCopy("\n"); + } + psFree(stack2Where); + + // don't queue for stacks that have already been diffed unless requested + psString diffQuery = NULL; + if (! (reRun || newTemplates) ) { + psStringAppend(&diffQuery, "\n AND diff_id IS NULL"); + } else { + diffQuery = psStringCopy("\n"); + } + + // find the distinct set of data_groups and filters + psString query = pxDataGet("difftool_definestackstack_part0.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return(false); + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + if (!p_psDBRunQueryF(config->dbh, query, stack2Query, diffQuery, stack1Query, stack1Query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + break; + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + break; + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + break; + } + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } + + query = pxDataGet("difftool_definestackstack_part1.sql"); + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffRunTemp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Output row from query + bool mdok; // Status of MD lookup + + psString tess_id = psMetadataLookupStr(&mdok,row,"INPUT_tess_id"); + psString this_data_group = psMetadataLookupStr(&mdok,row,"INPUT_data_group"); + psString this_dist_group = psMetadataLookupStr(&mdok,row,"INPUT_dist_group"); + psString this_label = psMetadataLookupStr(&mdok,row,"INPUT_label"); + + psString this_stack1Query = psStringCopy(stack1Query); + + psString thisWhere = psDBGenerateWhereConditionSQL(row,NULL); + psStringSubstitute(&thisWhere,"stackRun.","INPUT_"); + psStringAppend(&this_stack1Query,"\n AND %s", thisWhere); + psFree(thisWhere); + +/* psTrace("difftool",1, query,stack2Query,diffQuery,this_stack1Query,this_stack1Query); */ + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + if (!p_psDBRunQueryF(config->dbh, query, stack2Query, diffQuery, this_stack1Query, this_stack1Query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(this_stack1Query); + + psArray *diff_id_output = p_psDBFetchResult(config->dbh); + if (!diff_id_output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + break; + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + break; + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + break; + } + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (!psArrayLength(diff_id_output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(diff_id_output); + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } + + // ok we've got one create the diffRun + diffRunRow *run = diffRunRowAlloc( + 0, // ID + "reg", // state + workdir, + label ? label : this_label, + data_group ? data_group : this_data_group, + dist_group ? dist_group : this_dist_group, + reduction, + NULL, // dvodb + registered, + tess_id, + false, // bothways + false, // exposure + 0, // magicked + note + ); + // Commit to database + if (!diffRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + // diffRunPrintObject(stdout,run,1); + run->diff_id = psDBLastInsertID(config->dbh); + for (long j = 0; j < diff_id_output->n; j++) { + psMetadata *input_row = diff_id_output->data[j]; // Output row from query + bool mdok; + psString in_skycell_id = psMetadataLookupStr(&mdok,input_row,"skycell_id"); + psS64 in_input_stack_id = psMetadataLookupS64(&mdok,input_row,"stack_id"); + psS64 in_template_stack_id = psMetadataLookupS64(&mdok,input_row,"max_stack_id"); + psString in_tess_id = psMetadataLookupStr(&mdok,input_row,"tess_id"); + psTrace("difftool",1,"%s %ld %ld %s\n",in_skycell_id,in_input_stack_id,in_template_stack_id,in_tess_id); + diffInputSkyfileRow *input = diffInputSkyfileRowAlloc( + run->diff_id, // ID + in_skycell_id, + 0, // warp1_id + in_input_stack_id, // stack1 + 0, // warp2_id + in_template_stack_id, // stack2 + in_tess_id, + 0 // diff_skyfile_id + ); + // Commit to database the input + if (!diffInputSkyfileInsertObject(config->dbh, input)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(input); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + // diffInputSkyfilePrintObject(stdout,input,1); + psFree(input); + } + psFree(run); + } + psFree(query); + psFree(stack1Query); + psFree(stack2Query); + psFree(diffQuery); + psFree(output); + + return(true); +} static bool pendingcleanuprunMode(pxConfig *config) @@ -1840,5 +2100,5 @@ psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "=="); psString query = pxDataGet("difftool_pendingcleanuprun.sql"); @@ -1847,9 +2107,17 @@ return false; } - + // pxDataGet/psStringSubstitute workaround + psString queryCopy = psStringCopy(query); + psFree(query); + query = queryCopy; + if (where && psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&whereClause, " AND "); + psStringSubstitute(&query,whereClause,"@INNERCONSTRAINT@"); psFree(whereClause); + } + else { + psStringSubstitute(&query,NULL,"@INNERCONSTRAINT@"); } psFree(where); @@ -1861,6 +2129,7 @@ psFree(limitString); } - // fprintf(stderr,"%s",query); + + //fprintf(stderr,"%s",query); if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: /trunk/ippTools/src/difftool.h =================================================================== --- /trunk/ippTools/src/difftool.h (revision 26926) +++ /trunk/ippTools/src/difftool.h (revision 26927) @@ -37,4 +37,5 @@ DIFFTOOL_MODE_DEFINEWARPSTACK, DIFFTOOL_MODE_DEFINEWARPWARP, + DIFFTOOL_MODE_DEFINESTACKSTACK, DIFFTOOL_MODE_PENDINGCLEANUPRUN, DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, Index: /trunk/ippTools/src/difftoolConfig.c =================================================================== --- /trunk/ippTools/src/difftoolConfig.c (revision 26926) +++ /trunk/ippTools/src/difftoolConfig.c (revision 26927) @@ -253,4 +253,27 @@ psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + // -definestackstack + psMetadata *definestackstackArgs = psMetadataAlloc(); + // stack id and exp id searches seem less useful here + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-input_label", 0, "search by stack label for input", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-template_label", 0, "search by stack label for template", NULL); + psMetadataAddF32(definestackstackArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definestackstackArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false); + psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-available", 0, "define new run even if no stacks available for some skycells", false); + psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false); + psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-pretend", 0, "list results but do not queue", false); + + // -pendingcleanuprun psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); @@ -327,4 +350,5 @@ PXOPT_ADD_MODE("-definewarpstack", "", DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackArgs); PXOPT_ADD_MODE("-definewarpwarp", "", DIFFTOOL_MODE_DEFINEWARPWARP, definewarpwarpArgs); + PXOPT_ADD_MODE("-definestackstack", "", DIFFTOOL_MODE_DEFINESTACKSTACK, definestackstackArgs); PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs);