Index: branches/czw_branch/20101203/ippTools/configure.ac =================================================================== --- branches/czw_branch/20101203/ippTools/configure.ac (revision 30014) +++ branches/czw_branch/20101203/ippTools/configure.ac (revision 30118) @@ -1,5 +1,5 @@ AC_PREREQ(2.61) -AC_INIT([ipptools], [1.1.65], [ipp-support@ifa.hawaii.edu]) +AC_INIT([ipptools], [1.1.66], [ipp-support@ifa.hawaii.edu]) AC_CONFIG_SRCDIR([autogen.sh]) @@ -18,5 +18,5 @@ PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0]) PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) -PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.65]) +PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.66]) PKG_CHECK_MODULES([PPSTAMP], [ppstamp >= 0.1.1]) Index: branches/czw_branch/20101203/ippTools/share/pubtool_definerun.sql =================================================================== --- branches/czw_branch/20101203/ippTools/share/pubtool_definerun.sql (revision 30014) +++ branches/czw_branch/20101203/ippTools/share/pubtool_definerun.sql (revision 30118) @@ -3,5 +3,6 @@ client_id, stage_id, - src_label + src_label, + output_format FROM ( -- Get diffs to publish @@ -9,5 +10,6 @@ client_id, diff_id AS stage_id, - diffRun.label AS src_label + diffRun.label AS src_label, + output_format FROM publishClient JOIN diffRun @@ -28,5 +30,6 @@ client_id, cam_id AS stage_id, - camRun.label AS src_label + camRun.label AS src_label, + output_format FROM publishClient JOIN camRun @@ -43,5 +46,6 @@ client_id, diff_phot_id AS stage_id, - diffPhotRun.label AS src_label + diffPhotRun.label AS src_label, + output_format FROM publishClient JOIN diffPhotRun Index: branches/czw_branch/20101203/ippTools/share/pubtool_pending.sql =================================================================== --- branches/czw_branch/20101203/ippTools/share/pubtool_pending.sql (revision 30014) +++ branches/czw_branch/20101203/ippTools/share/pubtool_pending.sql (revision 30118) @@ -9,4 +9,5 @@ publishClient.stage, publishClient.workdir, + publishClient.output_format, diffRun.diff_id AS stage_id, rawExp.camera, @@ -35,4 +36,5 @@ publishClient.stage, publishClient.workdir, + publishClient.output_format, camRun.cam_id AS stage_id, rawExp.camera, @@ -56,4 +58,5 @@ publishClient.stage, publishClient.workdir, + publishClient.output_format, diffPhotRun.diff_phot_id AS stage_id, rawExp.camera, Index: branches/czw_branch/20101203/ippTools/share/pxadmin_create_tables.sql =================================================================== --- branches/czw_branch/20101203/ippTools/share/pxadmin_create_tables.sql (revision 30014) +++ branches/czw_branch/20101203/ippTools/share/pxadmin_create_tables.sql (revision 30118) @@ -1645,5 +1645,8 @@ workdir VARCHAR(255) NOT NULL, -- working directory comment VARCHAR(255), -- for human memory - PRIMARY KEY(client_id) + name varchar(64) default NULL, -- unique client_id verbose identifier + output_format SMALLINT NOT NULL default 1, -- format output versioning + PRIMARY KEY(client_id), + UNIQUE KEY name (name) ) ENGINE=innodb DEFAULT CHARSET=latin1; Index: branches/czw_branch/20101203/ippTools/share/regtool_pendingexp.sql =================================================================== --- branches/czw_branch/20101203/ippTools/share/regtool_pendingexp.sql (revision 30014) +++ branches/czw_branch/20101203/ippTools/share/regtool_pendingexp.sql (revision 30118) @@ -1,49 +1,54 @@ SELECT DISTINCT - exp_id, - tmp_exp_name, - tmp_camera, - tmp_telescope, - state, - workdir, - workdir_state, - reduction, - dvodb, - tess_id, - end_stage, - label, - camera, - filter, - obs_mode, - obs_group, - epoch, - dateobs -FROM - (SELECT - newExp.*, - newImfile.tmp_class_id, - rawImfile.tmp_class_id as raw_tmp_class_id, - rawImfile.camera, - rawImfile.filter, - rawImfile.dateobs, - rawImfile.obs_mode, - rawImfile.obs_group - FROM newExp - JOIN newImfile - USING(exp_id) - LEFT JOIN rawExp - USING(exp_id) - LEFT JOIN rawImfile - ON newImfile.exp_id = rawImfile.exp_id - AND newImfile.tmp_class_id = rawImfile.tmp_class_id - WHERE - newExp.state = 'run' - AND rawExp.exp_id IS NULL - AND rawImfile.data_state = 'full' --- where hook %s - GROUP BY - newExp.exp_id - HAVING - COUNT(newImfile.tmp_class_id) = COUNT(rawImfile.tmp_class_id) - AND SUM(rawImfile.fault) = 0 + exp_id, + tmp_exp_name, + tmp_camera, + tmp_telescope, + state, + workdir, + workdir_state, + reduction, + dvodb, + tess_id, + end_stage, + label, + camera, + filter, + obs_mode, + obs_group, + epoch, + dateobs +FROM + ( + select DISTINCT * from + ( + select newExp.*,count(newImfile.tmp_class_id) AS new_count + from newExp + LEFT JOIN newImfile USING (exp_id) + LEFT JOIN rawExp USING (exp_id) + where + newExp.state = 'run' AND rawExp.exp_id IS NULL + GROUP BY newExp.exp_id + ) AS NEWEXPOSURES + JOIN + ( + select DISTINCT newExp.exp_id, + rawImfile.camera, + rawImfile.filter, + rawImfile.dateobs, + rawImfile.obs_mode, + rawImfile.obs_group, + count(rawImfile.tmp_class_id) AS raw_count + from newExp + LEFT JOIN rawExp USING (exp_id) + LEFT JOIN rawImfile USING (exp_id) + where + newExp.state = 'run' AND rawExp.exp_id IS NULL + AND rawImfile.data_state = 'full' + -- where hook %s + GROUP BY newExp.exp_id + HAVING SUM(rawImfile.fault) = 0 + ) AS RAWEXPOSURES + USING (exp_id) + WHERE raw_count = new_count + ) AS Foo -- limit hook %s - ) as Foo Index: branches/czw_branch/20101203/ippTools/src/dettool.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/dettool.c (revision 30014) +++ branches/czw_branch/20101203/ippTools/src/dettool.c (revision 30118) @@ -572,4 +572,6 @@ PXOPT_COPY_F64(config->args, where, "-select_moon_phase_min", "moon_phase", ">="); PXOPT_COPY_F64(config->args, where, "-select_moon_phase_max", "moon_phase", "<="); + + PXOPT_COPY_STR(config->args, where, "-select_state", "state", "="); PXOPT_COPY_STR(config->args, where, "-comment", "comment", "LIKE"); Index: branches/czw_branch/20101203/ippTools/src/dettoolConfig.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/dettoolConfig.c (revision 30014) +++ branches/czw_branch/20101203/ippTools/src/dettoolConfig.c (revision 30118) @@ -138,4 +138,5 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_phase_min", 0, "define min moon phase", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_phase_max", 0, "define max moon phase", NAN); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_state", 0, "select by state (default is 'full')", "full"); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); Index: branches/czw_branch/20101203/ippTools/src/difftool.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/difftool.c (revision 30014) +++ branches/czw_branch/20101203/ippTools/src/difftool.c (revision 30118) @@ -483,5 +483,5 @@ psFree(where); - psStringAppend(&query, "\nORDER by priority DESC, diff_id"); + psStringAppend(&query, "\nORDER by priority DESC, diff_id, skycell_id"); // treat limit == 0 as "no limit" @@ -1978,4 +1978,5 @@ PXOPT_COPY_STR(config->args, stack2Where, "-skycell_id", "stackRun.skycell_id", "=="); PXOPT_COPY_STR(config->args, stack1Where, "-input_label", "stackRun.label","=="); + PXOPT_COPY_STR(config->args, stack1Where, "-input_data_group", "stackRun.data_group","=="); PXOPT_COPY_STR(config->args, stack2Where, "-template_label", "stackRun.label","=="); PXOPT_COPY_F32(config->args, stack1Where, "-good_frac", "stackSumSkyfile.good_frac", ">="); @@ -2042,8 +2043,4 @@ } - psString queryCopy = psStringCopy(query); - psFree(query); - query = queryCopy; - psStringSubstitute(&query, stack1Query, "@STACK1_QUERY@"); psStringSubstitute(&query, stack2Query, "@STACK2_QUERY@"); @@ -2120,9 +2117,4 @@ psStringAppend(&this_stack1Query,"AND %s", thisWhere); psFree(thisWhere); - - psString queryCopy = psStringCopy(query); - psFree(query); - query = queryCopy; - psStringSubstitute(&query, this_stack1Query, "@STACK1_QUERY@"); Index: branches/czw_branch/20101203/ippTools/src/difftoolConfig.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/difftoolConfig.c (revision 30014) +++ branches/czw_branch/20101203/ippTools/src/difftoolConfig.c (revision 30118) @@ -336,4 +336,5 @@ psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-input_label", 0, "search by stack label for input", NULL); psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-template_label", 0, "search by stack label for template", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-input_data_group", 0, "search by stack data_group for input", NULL); psMetadataAddF32(definestackstackArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); Index: branches/czw_branch/20101203/ippTools/src/magicdstool.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/magicdstool.c (revision 30014) +++ branches/czw_branch/20101203/ippTools/src/magicdstool.c (revision 30118) @@ -1155,6 +1155,6 @@ psMetadata *where = psMetadataAlloc(); - // new state - PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", false, false); + // new state (required + PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", true, false); // old state (required) PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); Index: branches/czw_branch/20101203/ippTools/src/magicdstoolConfig.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/magicdstoolConfig.c (revision 30014) +++ branches/czw_branch/20101203/ippTools/src/magicdstoolConfig.c (revision 30118) @@ -163,5 +163,5 @@ // -clearstatefaults psMetadata *clearstatefaultsArgs = psMetadataAlloc(); - psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL); + psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state (required)", NULL); psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-state", 0, "search by state (required)", NULL); psMetadataAddS64(clearstatefaultsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magictool de-streak ID", 0); Index: branches/czw_branch/20101203/ippTools/src/pstamptool.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/pstamptool.c (revision 30014) +++ branches/czw_branch/20101203/ippTools/src/pstamptool.c (revision 30118) @@ -950,9 +950,12 @@ PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); + + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_STR(config->args, where, "-component", "component", "=="); + PXOPT_COPY_S32(config->args, where, "-fault", "pstampDependent.fault", "=="); + pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE"); - // if (fault_count) { - PXOPT_COPY_S32(config->args, where, "-fault_count", "pstampDependent.fault_count", ">="); - // } + PXOPT_COPY_S32(config->args, where, "-fault_count", "pstampDependent.fault_count", ">="); // XXX: How about selecting by pstampRequest.label? No. That is too dangerous by itself. Index: branches/czw_branch/20101203/ippTools/src/pstamptoolConfig.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/pstamptoolConfig.c (revision 30014) +++ branches/czw_branch/20101203/ippTools/src/pstamptoolConfig.c (revision 30118) @@ -178,4 +178,6 @@ psMetadataAddS64(stopdependentjobArgs, PS_LIST_TAIL, "-job_id", 0, "job_id of jobs to update", 0); psMetadataAddS64(stopdependentjobArgs, PS_LIST_TAIL, "-dep_id", 0, "dep_id of jobs to update", 0); + psMetadataAddS64(stopdependentjobArgs, PS_LIST_TAIL, "-stage_id", 0, "stage_id of jobs to update", 0); + psMetadataAddStr(stopdependentjobArgs, PS_LIST_TAIL, "-component", 0, "component of jobs to update", NULL); psMetadataAddS16(stopdependentjobArgs, PS_LIST_TAIL, "-fault", 0, "current value for dependent fault", 0); psMetadataAddS32(stopdependentjobArgs, PS_LIST_TAIL, "-fault_count", 0, "select by fault_count (>=)", 0); Index: branches/czw_branch/20101203/ippTools/src/pubtool.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/pubtool.c (revision 30014) +++ branches/czw_branch/20101203/ippTools/src/pubtool.c (revision 30118) @@ -96,6 +96,8 @@ PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); PXOPT_LOOKUP_BOOL(unmagicked, config->args, "-unmagicked", false); - - if (!publishClientInsert(config->dbh, 0, 0, product, stage, !unmagicked, workdir, comment)) { + PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); + PXOPT_LOOKUP_S16(output_format, config->args, "-output_format", false, false); + + if (!publishClientInsert(config->dbh, 0, 0, product, stage, !unmagicked, workdir, comment, name, output_format)) { psError(PS_ERR_UNKNOWN, false, "Database error"); return false; @@ -313,4 +315,5 @@ PXOPT_COPY_STR(config->args, where, "-stage", "publishClient.stage", "=="); PXOPT_COPY_STR(config->args, where, "-comment", "publishClient.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-name", "publishClient.name", "LIKE"); pxAddLabelSearchArgs(config, where, "-label", "publishRun.label", "=="); Index: branches/czw_branch/20101203/ippTools/src/pubtoolConfig.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/pubtoolConfig.c (revision 30014) +++ branches/czw_branch/20101203/ippTools/src/pubtoolConfig.c (revision 30118) @@ -83,4 +83,5 @@ psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, "use simple output format?", false); psMetadataAddU64(pendingArgs, PS_LIST_TAIL, "-limit", 0, "limit result set", 0); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-name", 0, "search on client name", NULL); // -add Index: branches/czw_branch/20101203/ippTools/src/regtool.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/regtool.c (revision 30014) +++ branches/czw_branch/20101203/ippTools/src/regtool.c (revision 30118) @@ -300,5 +300,5 @@ psStringSubstitute(&query,date,"@DATE@"); - fprintf(stderr,"%s",query); + // fprintf(stderr,"%s",query); if (!p_psDBRunQuery(config->dbh, query)) { @@ -349,5 +349,8 @@ psMetadataAddStr(row,PS_LIST_TAIL,"previous_class_id",PS_META_REPLACE,"",previous_class_id); } - + // class_id = NULL sorts to the top of the list, so skip those (incomplete downloads) + if (!this_class_id) { + continue; + } // Determine if we've crossed a class_id boundary, as this resets the bits. if (previous_class_id) { @@ -370,7 +373,9 @@ if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)|| (psMetadataLookupS32(NULL,row,"is_registered") != 1)) { + // printf("I claim this isn't downloaded or registered? %s %s\n",this_uri,this_class_id); ok_to_burn = false; } if (already_burned == false) { + // printf("already_burned looks false %s %s\n",this_uri,this_class_id); ok_to_burn = false; } @@ -398,5 +403,5 @@ } - // printf("STATUS: %s %s %s %s (%d %d)\n",this_uri,previous_uri,this_class_id,previous_class_id,ok_to_burn,already_burned); + // printf("STATUS: %s %s %s %s (%d %d) %d %d %d\n",this_uri,previous_uri,this_class_id,previous_class_id,ok_to_burn,already_burned,psMetadataLookupS32(NULL,row,"burntool_state"),psMetadataLookupS32(NULL,row,"is_registered"),psMetadataLookupS32(NULL,row,"is_downloaded")); // If the state of this imfile is not "pending_burntool" then we can't burn it.