Index: /branches/bills_081204/dbconfig/changes.txt =================================================================== --- /branches/bills_081204/dbconfig/changes.txt (revision 20953) +++ /branches/bills_081204/dbconfig/changes.txt (revision 20954) @@ -131,5 +131,5 @@ ALTER TABLE warpRun ADD CONSTRAINT FOREIGN KEY(cam_id) REFERENCES camRun(cam_id); -ALTER TABLE warpRun ADD COLUMN magiced TINYINT AFTER registered; +ALTER TABLE warpRun ADD COLUMN magicked TINYINT AFTER registered; ALTER TABLE warpSkyCellMap DROP FOREIGN KEY warpSkyCellMap_ibfk_1; UPDATE warpRun JOIN warpInputExp USING(warp_id) SET warpRun.cam_id = @@ -723,5 +723,5 @@ ALTER TABLE warpRun ADD CONSTRAINT FOREIGN KEY(cam_id) REFERENCES camRun(cam_id); -ALTER TABLE warpRun ADD COLUMN magiced TINYINT AFTER registered; +ALTER TABLE warpRun ADD COLUMN magicked TINYINT AFTER registered; ALTER TABLE warpSkyCellMap DROP FOREIGN KEY warpSkyCellMap_ibfk_1; UPDATE warpRun JOIN warpInputExp USING(warp_id) SET warpRun.cam_id = @@ -1049,11 +1049,11 @@ UPDATE stackSumSkyfile SET data_state = 'full'; -alter table diffRun add column label varchar(64) after workdir; -alter table diffRun add column reduction varchar(64) after label; - -ALTER TABLE diffSkyfile add column data_state varchar(64) after path_base; +ALTER TABLE diffRun ADD COLUMN label VARCHAR(64) AFTER workdir; +ALTER TABLE diffRun ADD COLUMN reduction VARCHAR(64) AFTER label; + +ALTER TABLE diffSkyfile ADD COLUMN data_state VARCHAR(64) AFTER path_base; UPDATE diffSkyfile SET data_state = 'full'; -ALTER TABLE warpSkyfile add column data_state varchar(64) after path_base; +ALTER TABLE warpSkyfile ADD COLUMN data_state VARCHAR(64) AFTER path_base; UPDATE warpSkyfile SET data_state = 'full'; @@ -1309,4 +1309,60 @@ ALTER TABLE magicDSRun ADD COLUMN cam_id BIGINT AFTER stage_id; +-- Version: 1.1.47 is this correct? + + +-- fix order of workdir to match pxadmin_create_tables.sql +ALTER TABLE diffRun CHANGE COLUMN workdir workdir VARCHAR(255) AFTER state; + +ALTER TABLE diffRun ADD COLUMN exp_id BIGINT; +ALTER TABLE diffRun ADD FOREIGN KEY (exp_id) REFERENCES rawExp(exp_id); +ALTER TABLE diffRun DROP COLUMN skycell_id; + + +-- drop exiting foreign key constraints +ALTER TABLE diffInputSkyfile drop FOREIGN KEY diffInputSkyfile_ibfk_1; +ALTER TABLE diffInputSkyfile drop FOREIGN KEY diffInputSkyfile_ibfk_2; +ALTER TABLE diffInputSkyfile drop FOREIGN KEY diffInputSkyfile_ibfk_3; + +ALTER TABLE diffInputSkyfile DROP COLUMN kind; +ALTER TABLE diffInputSkyfile CHANGE COLUMN skycell_id skycell_id VARCHAR(64) AFTER diff_id; + +-- run script collapse_diffinputs.pl +DELETE FROM diffInputSkyfile where template = 1; + +ALTER TABLE diffInputSkyfile DROP PRIMARY KEY; +ALTER TABLE diffInputSkyfile ADD PRIMARY KEY (diff_id, skycell_id); +ALTER TABLE diffInputSkyfile DROP COLUMN template; + +ALTER TABLE diffInputSkyfile CHANGE COLUMN warp_id warp1 BIGINT; +ALTER TABLE diffInputSkyfile ADD COLUMN stack1 BIGINT AFTER warp1, ADD KEY(stack1); +ALTER TABLE diffInputSkyfile ADD COLUMN warp2 BIGINT AFTER stack1, ADD KEY(warp2); +ALTER TABLE diffInputSkyfile CHANGE COLUMN stack_id stack2 BIGINT AFTER warp2; +ALTER TABLE diffInputSkyfile ADD FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id); + +# need to check these +ALTER TABLE diffInputSkyfile ADD FOREIGN KEY (warp1, skycell_id, tess_id) REFERENCES warpSkyfile(warp_id, skycell_id, tess_id); +ALTER TABLE diffInputSkyfile ADD FOREIGN KEY (warp2, skycell_id, tess_id) REFERENCES warpSkyfile(warp_id, skycell_id, tess_id); + +ALTER TABLE diffInputSkyfile ADD FOREIGN KEY (stack1) REFERENCES stackRun(stack_id); +ALTER TABLE diffInputSkyfile ADD FOREIGN KEY (stack2) REFERENCES stackRun(stack_id); + +-- add unique ids for each image and flags that indicate whether the image has been +-- magic destreaked +ALTER TABLE diffSkyfile ADD COLUMN skycell_id VARCHAR(64) AFTER diff_id; +ALTER TABLE diffSkyfile DROP PRIMARY KEY, ADD PRIMARY KEY (diff_id, skycell_id); +ALTER TABLE diffSkyfile ADD COLUMN diff_image_id BIGINT NOT NULL AUTO_INCREMENT, ADD KEY(diff_image_id); +ALTER TABLE diffSkyfile ADD COLUMN magicked TINYINT; + +ALTER TABLE warpRun DROP COLUMN magiced; + +ALTER TABLE warpSkyfile ADD COLUMN warp_image_id BIGINT NOT NULL AUTO_INCREMENT, ADD KEY(warp_image_id); +ALTER TABLE warpSkyfile ADD COLUMN magicked TINYINT; + +ALTER TABLE chipProcessedImfile ADD COLUMN chip_image_id BIGINT NOT NULL AUTO_INCREMENT, ADD KEY(chip_image_id); +ALTER TABLE chipProcessedImfile ADD COLUMN magicked TINYINT; + +ALTER TABLE rawImfile ADD COLUMN raw_image_id BIGINT NOT NULL AUTO_INCREMENT, ADD KEY(raw_image_id); +ALTER TABLE rawImfile ADD COLUMN magicked TINYINT; @@ -1314,5 +1370,2 @@ ALTER TABLE magicRun ADD COLUMN diff_id BIGINT AFTER exp_id; ALTER TABLE magicRun ADD CONSTRAINT FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id); - -ALTER TABLE magicInputSkyfile DROP FOREIGN KEY magicInputSkyfile_ibfk_2; -ALTER TABLE magicInputSkyfile DROP column diff_id; Index: /branches/bills_081204/dbconfig/chip.md =================================================================== --- /branches/bills_081204/dbconfig/chip.md (revision 20953) +++ /branches/bills_081204/dbconfig/chip.md (revision 20954) @@ -81,4 +81,6 @@ path_base STR 255 fault S16 0 # Key NOT NULL + chip_image_id S64 0 # Key AUTO_INCREMENT + magicked BOOL f END Index: /branches/bills_081204/dbconfig/diff.md =================================================================== --- /branches/bills_081204/dbconfig/diff.md (revision 20953) +++ /branches/bills_081204/dbconfig/diff.md (revision 20954) @@ -1,14 +1,14 @@ -# $Id: diff.md,v 1.14.8.1 2008-12-04 23:44:33 bills Exp $ +# $Id: diff.md,v 1.14.8.2 2008-12-11 04:11:39 bills Exp $ diffRun METADATA diff_id S64 0 # Primary Key AUTO_INCREMENT state STR 64 # Key + workdir STR 255 label STR 64 # Key reduction STR 64 # Reduction class - workdir STR 255 dvodb STR 255 registered TAI NULL + tess_id STR 64 # Key exp_id S64 0 # fkey(exp_id) ref rawExp(exp_id) - tess_id STR 64 # Key END @@ -19,9 +19,9 @@ diffInputSkyfile METADATA diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) + skycell_id STR 64 # Primary Key warp1 S64 0 # fkey(warp1, skycell_id, tess_id) ref warpSkyfile(warp_id, skycell_id, tess_id) + stack1 S64 0 # fkey(stack1) ref stackSumSkyfile(stack_id) warp2 S64 0 # fkey(warp2, skycell_id, tess_id) ref warpSkyfile(warp_id, skycell_id, tess_id) - stack1 S64 0 # fkey(stack1) ref stackSumSkyfile(stack_id) stack2 S64 0 # fkey(stack2) ref stackSumSkyfile(stack_id) - skycell_id STR 64 # Key tess_id STR 64 # Key END @@ -29,5 +29,5 @@ diffSkyfile METADATA diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) - skycell_id STR 64 # Key + skycell_id STR 64 # uri STR 255 path_base STR 255 @@ -52,3 +52,5 @@ good_frac F32 0.0 # Key fault S16 0 # Key + diff_image_id S64 0 # Key NOT NULL AUTO_INCREMENT + magicked BOOL f END Index: /branches/bills_081204/dbconfig/magic.md =================================================================== --- /branches/bills_081204/dbconfig/magic.md (revision 20953) +++ /branches/bills_081204/dbconfig/magic.md (revision 20954) @@ -1,3 +1,3 @@ -# $Id: magic.md,v 1.12.2.1 2008-12-04 23:44:33 bills Exp $ +# $Id: magic.md,v 1.12.2.2 2008-12-11 04:11:39 bills Exp $ ### Fault in magicRun indicates that the processing tree failed @@ -17,4 +17,5 @@ magicInputSkyfile METADATA magic_id S64 0 # Primary Key fkey(magic_id) ref magicRun(magic_id) + diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) node STR 64 # END Index: /branches/bills_081204/dbconfig/tasks.md =================================================================== --- /branches/bills_081204/dbconfig/tasks.md (revision 20953) +++ /branches/bills_081204/dbconfig/tasks.md (revision 20954) @@ -1,3 +1,3 @@ -# $Id: tasks.md,v 1.158 2008-11-09 23:51:01 price Exp $ +# $Id: tasks.md,v 1.158.2.1 2008-12-11 04:11:39 bills Exp $ # this table records all exposure ID ever seen from the summit @@ -239,3 +239,4 @@ fault S16 0 # Key NOT NULL epoch UTC 0001-01-01T00:00:00Z -END + magicked BOOL f +END Index: /branches/bills_081204/dbconfig/warp.md =================================================================== --- /branches/bills_081204/dbconfig/warp.md (revision 20953) +++ /branches/bills_081204/dbconfig/warp.md (revision 20954) @@ -1,3 +1,3 @@ -# $Id: warp.md,v 1.19 2008-10-07 00:01:28 price Exp $ +# $Id: warp.md,v 1.19.10.1 2008-12-11 04:11:39 bills Exp $ # @@ -11,5 +11,5 @@ warpRun METADATA warp_id S64 0 # Primary Key AUTO_INCREMENT - fake_id S64 0 # Key INDEX(warp_id, fake_id) fkey(fake_id) ref camProcessedExp(fake_id) + fake_id S64 0 # Key INDEX(warp_id, fake_id) fkey(fake_id) ref camProcessedExp(fake_id) mode STR 64 # Key state STR 64 # Key @@ -22,5 +22,4 @@ registered TAI NULL # if magic is T then look for the exp_id in the magic output tables - magiced BOOL f # Key END @@ -52,4 +51,6 @@ ignored BOOL f # Key fault S16 0 # Key + warp_image_id S64 0 # Key AUTO_INCREMENT + magicked BOOL f END Index: /branches/bills_081204/ippTasks/diff.pro =================================================================== --- /branches/bills_081204/ippTasks/diff.pro (revision 20953) +++ /branches/bills_081204/ippTasks/diff.pro (revision 20954) @@ -98,5 +98,5 @@ task.exit 0 # convert 'stdout' to book format - ipptool2book stdout diffSkyfile -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT + ipptool2book stdout diffSkyfile -key diff_id:skycell_id -uniq -setword dbname $options:0 -setword pantaskState INIT if ($VERBOSE > 2) book listbook diffSkyfile @@ -214,5 +214,5 @@ task.exit 0 # convert 'stdout' to book format - ipptool2book stdout diffCleanup -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT + ipptool2book stdout diffCleanup -key diff_id:skycell_id -uniq -setword dbname $options:0 -setword pantaskState INIT if ($VERBOSE > 2) book listbook diffCleanup Index: /branches/bills_081204/ippTools/share/Makefile.am =================================================================== --- /branches/bills_081204/ippTools/share/Makefile.am (revision 20953) +++ /branches/bills_081204/ippTools/share/Makefile.am (revision 20954) @@ -72,4 +72,7 @@ difftool_completed_runs.sql \ difftool_definebyquery.sql \ + difftool_definebyquery_part1.sql \ + difftool_definebyquery_part2.sql \ + difftool_definebyquery_temp_create.sql \ difftool_donecleanup.sql \ difftool_inputskyfile.sql \ Index: /branches/bills_081204/ippTools/share/difftool_completed_runs.sql =================================================================== --- /branches/bills_081204/ippTools/share/difftool_completed_runs.sql (revision 20953) +++ /branches/bills_081204/ippTools/share/difftool_completed_runs.sql (revision 20954) @@ -3,14 +3,15 @@ FROM ( SELECT + COUNT(diffInputSkyfile.skycell_id), COUNT(diffSkyfile.skycell_id), diffSkyfile.* FROM diffRun JOIN diffInputSkyfile USING(diff_id) - LEFT JOIN diffSkyfile USING(diff_id) + LEFT JOIN diffSkyfile USING(diff_id, skycell_id) WHERE diffRun.state = 'new' GROUP BY - diffSkyfile.diff_id + diffInputSkyfile.diff_id HAVING - COUNT(diffInputSkyfile.diff_id) = COUNT(diffSkyfile.diff_id) + COUNT(diffInputSkyfile.skycell_id) = COUNT(diffSkyfile.skycell_id) AND SUM(diffSkyfile.fault) = 0 ) as Foo Index: /branches/bills_081204/ippTools/share/difftool_todiffskyfile.sql =================================================================== --- /branches/bills_081204/ippTools/share/difftool_todiffskyfile.sql (revision 20953) +++ /branches/bills_081204/ippTools/share/difftool_todiffskyfile.sql (revision 20954) @@ -56,4 +56,5 @@ LEFT JOIN diffSkyfile ON diffInputSkyfile.diff_id = diffSkyfile.diff_id + AND diffInputSkyfile.skycell_id = diffSkyfile.skycell_id WHERE -- Ready to be processed Index: /branches/bills_081204/ippTools/share/magictool_definebyquery_insert.sql =================================================================== --- /branches/bills_081204/ippTools/share/magictool_definebyquery_insert.sql (revision 20953) +++ /branches/bills_081204/ippTools/share/magictool_definebyquery_insert.sql (revision 20954) @@ -3,4 +3,5 @@ SELECT @MAGIC_ID@, -- Update this with the appropriate magic_id + diff_id, skycell_id FROM diffSkyfile Index: /branches/bills_081204/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/bills_081204/ippTools/share/pxadmin_create_tables.sql (revision 20953) +++ /branches/bills_081204/ippTools/share/pxadmin_create_tables.sql (revision 20954) @@ -238,7 +238,10 @@ fault SMALLINT NOT NULL, epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + raw_image_id BIGINT AUTO_INCREMENT, + magicked TINYINT, PRIMARY KEY(exp_id, class_id), KEY(tmp_class_id), KEY(fault), + KEY(raw_image_id), UNIQUE KEY(exp_id, tmp_class_id), FOREIGN KEY (exp_id, tmp_class_id) @@ -330,7 +333,10 @@ path_base VARCHAR(255), fault SMALLINT NOT NULL, + chip_image_id BIGINT AUTO_INCREMENT, + magicked BIGINT, PRIMARY KEY(chip_id, exp_id, class_id), KEY(data_state), KEY(fault), + KEY(chip_image_id), FOREIGN KEY (chip_id, exp_id) REFERENCES chipRun(chip_id, exp_id), @@ -815,5 +821,4 @@ end_stage VARCHAR(64), registered DATETIME, - magiced TINYINT, PRIMARY KEY(warp_id), KEY(warp_id), @@ -824,5 +829,4 @@ KEY(label), KEY(end_stage), - KEY(magiced), INDEX(warp_id, fake_id), FOREIGN KEY (fake_id) @@ -861,7 +865,10 @@ ignored TINYINT, fault SMALLINT, + warp_image_id BIGINT AUTO_INCREMENT, + magicked TINYINT, PRIMARY KEY(warp_id, skycell_id, tess_id), KEY(good_frac), KEY(ignored), KEY(fault), + KEY(warp_image_id), FOREIGN KEY (warp_id, skycell_id, tess_id) REFERENCES warpSkyCellMap(warp_id, skycell_id, tess_id) @@ -941,21 +948,22 @@ dvodb VARCHAR(255), registered DATETIME, + tess_id VARCHAR(64), exp_id BIGINT, - tess_id VARCHAR(64), PRIMARY KEY(diff_id), KEY(diff_id), KEY(state), - KEY(tess_id) + KEY(tess_id), + FOREIGN KEY (exp_id) REFERENCES rawExp(exp_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE diffInputSkyfile ( diff_id BIGINT, + skycell_id VARCHAR(64), warp1 BIGINT, + stack1 BIGINT, warp2 BIGINT, - stack1 BIGINT, stack2 BIGINT, - skycell_id VARCHAR(64), tess_id VARCHAR(64), - PRIMARY KEY(diff_id), + PRIMARY KEY(diff_id, skycell_id), KEY(warp1), KEY(warp2), @@ -996,8 +1004,10 @@ good_frac FLOAT, fault SMALLINT, - PRIMARY KEY(diff_id), + diff_image_id BIGINT AUTO_INCREMENT, + magicked TINYINT, + PRIMARY KEY(diff_id, skycell_id), KEY(good_frac), KEY(fault), - KEY(skycell_id), + KEY(diff_image_id), FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1026,7 +1036,9 @@ CREATE TABLE magicInputSkyfile ( magic_id BIGINT, + diff_id BIGINT, node VARCHAR(64), - PRIMARY KEY(magic_id), - FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id) + PRIMARY KEY(magic_id, diff_id), + FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), + FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; Index: /branches/bills_081204/ippTools/share/warptool_tooverlap.sql =================================================================== --- /branches/bills_081204/ippTools/share/warptool_tooverlap.sql (revision 20953) +++ /branches/bills_081204/ippTools/share/warptool_tooverlap.sql (revision 20954) @@ -7,6 +7,5 @@ rawExp.camera, exp_id, - exp_tag, - warpRun.magiced + exp_tag FROM warpRun JOIN fakeRun Index: /branches/bills_081204/ippTools/src/chiptool.c =================================================================== --- /branches/bills_081204/ippTools/src/chiptool.c (revision 20953) +++ /branches/bills_081204/ippTools/src/chiptool.c (revision 20954) @@ -484,5 +484,7 @@ n_cr, path_base, - code + code, + 0, // chip_image_id + 0 // magic_ds_id )) { // rollback Index: /branches/bills_081204/ippTools/src/difftool.c =================================================================== --- /branches/bills_081204/ippTools/src/difftool.c (revision 20953) +++ /branches/bills_081204/ippTools/src/difftool.c (revision 20954) @@ -121,11 +121,11 @@ 0, // ID "reg", // state + workdir, label, reduction, - workdir, NULL, // dvodb registered, - exp_id, - tess_id + tess_id, + exp_id ); if (!run) { @@ -234,9 +234,9 @@ if (!diffInputSkyfileInsert(config->dbh, diff_id, + skycell_id, warp1 ? warp1 : PS_MAX_S64, // defined or NULL + stack1 ? stack1 : PS_MAX_S64, // defined or NULL warp2 ? warp2 : PS_MAX_S64, // defined or NULL - stack1 ? stack1 : PS_MAX_S64, // defined or NULL stack2 ? stack2 : PS_MAX_S64, // defined or NULL - skycell_id, tess_id )) { @@ -304,5 +304,5 @@ PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "=="); PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); @@ -531,5 +531,7 @@ hostname, good_frac, - code + code, + 0, // diff_image_id + 0 // magic_ds_id )) { if (!psDBRollback(config->dbh)) { @@ -795,11 +797,11 @@ 0, // ID "reg", // state + workdir, label, reduction, - workdir, NULL, // dvodb registered, - exp_id, - tess_id + tess_id, + exp_id ); @@ -826,9 +828,9 @@ if (!diffInputSkyfileInsert(config->dbh, run->diff_id, + skycell_id, input_warp_id, + input_stack_id, template_warp_id, - input_stack_id, template_stack_id, - skycell_id, tess_id )) { @@ -936,7 +938,9 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *expWhere = psMetadataAlloc(); psMetadata *warpWhere = psMetadataAlloc(); psMetadata *stackWhere = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, expWhere, "-exp_id", "exp_id", "=="); PXOPT_COPY_S64(config->args, warpWhere, "-warp_id", "warpRun.warp_id", "=="); PXOPT_COPY_STR(config->args, warpWhere, "-skycell_id", "warpRun.skycell_id", "=="); @@ -953,7 +957,9 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false); + PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(available, config->args, "-available", false); // find all things to queue - psString query = pxDataGet("difftool_definebyquery.sql"); + psString query = pxDataGet("difftool_definebyquery_part1.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -961,15 +967,16 @@ } - if (newTemplates) { - // Warps that haven't been diffed OR warps that can take a newer template in the diff - psStringAppend(&query, " WHERE (diff_id IS NULL OR best_stack_id > current_stack_id)"); - } else { - // Only warps that haven't been diffed - psStringAppend(&query, " WHERE diff_id IS NULL"); - } - psString warpQuery = NULL; psString stackQuery = NULL; - + psString expQuery = NULL; + + if (psListLength(expWhere->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(expWhere, NULL); + psStringAppend(&expQuery, "\n AND %s", whereClause); + psFree(whereClause); + } else { + expQuery = psStringCopy("\n"); + } + psFree(expWhere); if (psListLength(warpWhere->list)) { psString whereClause = psDBGenerateWhereConditionSQL(warpWhere, NULL); @@ -977,11 +984,22 @@ psFree(whereClause); } else { - warpQuery = psStringCopy(""); + warpQuery = psStringCopy("\n"); } psFree(warpWhere); + + if (!available) { + // diff what's available, even if warp run has some faults and is incomplete + psStringAppend(&warpQuery, " AND warpRun.state = 'full'"); + } + + // don't queue for exposures that have already been diff'd unless requested + psString diffQuery = NULL; + if (! (reRun || newTemplates) ) { + psStringAppend(&diffQuery, "\nAND diff_id IS NULL"); + } if (psListLength(stackWhere->list)) { psString whereClause = psDBGenerateWhereConditionSQL(stackWhere, NULL); - psStringAppend(&stackQuery, "\n AND %s", whereClause); + psStringAppend(&stackQuery, "\nAND %s", whereClause); psFree(whereClause); } else { @@ -990,7 +1008,8 @@ psFree(stackWhere); - psTrace("difftool", 1, query, warpQuery, stackQuery); - - if (!p_psDBRunQuery(config->dbh, query, warpQuery, stackQuery)) { + psTrace("difftool", 1, query, warpQuery, diffQuery, expQuery, stackQuery); + + + if (!p_psDBRunQuery(config->dbh, query, warpQuery, expQuery, diffQuery)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -1019,45 +1038,140 @@ } + // create temporary table + query = pxDataGet("difftool_definebyquery_temp_create.sql"); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + query = NULL; + + psString skycell_query = pxDataGet("difftool_definebyquery_part2.sql"); + psArray *list = psArrayAllocEmpty(16); // List of runs, to print long numGood = 0; // Number of good rows added + psS64 last_exp_id = 0; for (long i = 0; i < output->n; i++) { psMetadata *row = output->data[i]; // Output row from query - - // Selected parameters bool mdok; // Status of MD lookup - const char *skycell_id = psMetadataLookupStr(&mdok, row, "skycell_id"); - if (!mdok || !skycell_id) { - psWarning("skycell_id not found --- ignoring row %ld", i); + + // Take the first warp for each exposure. + // The list is sorted by exposure id and warp_id and the warps are in descending + // order. + psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "exp_id not found"); + return false; + } + if (exp_id == last_exp_id) { continue; } - const char *tess_id = psMetadataLookupStr(&mdok, row, "tess_id"); - if (!mdok || !tess_id) { - psWarning("tess_id not found --- ignoring row %ld", i); - continue; - } - psS64 stack_id = psMetadataLookupS64(&mdok, row, "best_stack_id"); - if (!mdok) { - psWarning("stack_id not found --- ignoring row %ld", i); - continue; + last_exp_id = exp_id; + + // clear temporary table + if (!p_psDBRunQuery(config->dbh, "DELETE FROM skycellsToDiff")) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + return false; } psS64 warp_id = psMetadataLookupS64(&mdok, row, "warp_id"); if (!mdok) { - psWarning("warp_id not found --- ignoring row %ld", i); + psError(PXTOOLS_ERR_PROG, false, "warp_id not found --- ignoring row %ld", i); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + return false; + } + psS64 skycell_count = psMetadataLookupS64(&mdok, row, "skycell_count"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "skycell_count not found"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + return false; + } + psString tess_id = psMetadataLookupStr(&mdok, row, "tess_id"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "tess_id not found"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + return false; + } + if (!p_psDBRunQuery(config->dbh, skycell_query, warp_id, warpQuery, stackQuery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + return false; + } + psS64 num = psDBAffectedRows(config->dbh); + + if (num == 0) { + psTrace("difftool", PS_LOG_INFO, "no skycells with stack found for %" PRId64, warp_id); continue; } - psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id"); - if (!mdok) { - psWarning("exp_id not found --- ignoring row %ld", i); + + if (!available && (num != skycell_count)) { + psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %" + PRId64 " need %" PRId64, num, skycell_count, warp_id); continue; } - if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction, warp_id, - PS_MAX_S64, PS_MAX_S64, stack_id, exp_id, config)) { - psWarning("Unable to add run for %s,%s,%" PRId64 ",%" PRId64, skycell_id, tess_id, - warp_id, stack_id); - psErrorClear(); - continue; - } - + // ok we've got one create the diffRun + diffRunRow *run = diffRunRowAlloc( + 0, // ID + "reg", // state + workdir, + label, + reduction, + NULL, // dvodb + registered, + tess_id, + exp_id + ); + + if (!diffRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return true; + } + run->diff_id = psDBLastInsertID(config->dbh); + + psStringAppend(&query, "UPDATE skycellsToDiff SET diff_id = %" PRId64, run->diff_id); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + psFree(query); + return false; + } + psFree(query); + query = NULL; + psStringAppend(&query, "INSERT INTO diffInputSkyfile SELECT * from skycellsToDiff"); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + psFree(query); + return false; + } + + if (!setdiffRunState(config, run->diff_id, "new")) { + psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, + run->diff_id); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + psFree(query); + return false; + } + + psArrayAdd(list, list->n, run); numGood++; } Index: /branches/bills_081204/ippTools/src/difftoolConfig.c =================================================================== --- /branches/bills_081204/ippTools/src/difftoolConfig.c (revision 20953) +++ /branches/bills_081204/ippTools/src/difftoolConfig.c (revision 20954) @@ -145,6 +145,6 @@ // -definebyquery psMetadata *definebyqueryArgs = psMetadataAlloc(); - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL); @@ -158,4 +158,6 @@ psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); Index: /branches/bills_081204/ippTools/src/magictool.c =================================================================== --- /branches/bills_081204/ippTools/src/magictool.c (revision 20953) +++ /branches/bills_081204/ippTools/src/magictool.c (revision 20954) @@ -362,4 +362,5 @@ config->dbh, magic_id, + diff_id, node ); Index: /branches/bills_081204/ippTools/src/pxwarp.c =================================================================== --- /branches/bills_081204/ippTools/src/pxwarp.c (revision 20953) +++ /branches/bills_081204/ippTools/src/pxwarp.c (revision 20954) @@ -147,6 +147,5 @@ tess_id, end_stage, - NULL, // registered - false // magiced + NULL // registered )) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: /branches/bills_081204/ippTools/src/regtool.c =================================================================== --- /branches/bills_081204/ippTools/src/regtool.c (revision 20953) +++ /branches/bills_081204/ippTools/src/regtool.c (revision 20954) @@ -299,5 +299,6 @@ hostname, code, - NULL + NULL, + 0 )) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: /branches/bills_081204/ippTools/src/warptool.c =================================================================== --- /branches/bills_081204/ippTools/src/warptool.c (revision 20953) +++ /branches/bills_081204/ippTools/src/warptool.c (revision 20954) @@ -134,5 +134,4 @@ PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); - PXOPT_LOOKUP_BOOL(magiced, config->args, "-magiced", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -154,6 +153,5 @@ tess_id, end_stage, - registered, - magiced + registered ); if (!warpRun) { @@ -246,5 +244,4 @@ PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); - PXOPT_LOOKUP_BOOL(magiced, config->args, "-magiced", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -942,5 +939,7 @@ ymax, !accept, - code + code, + 0, // warp_image_id + 0 // magic_ds_id )) { if (!psDBRollback(config->dbh)) { Index: /branches/bills_081204/ippTools/src/warptoolConfig.c =================================================================== --- /branches/bills_081204/ippTools/src/warptoolConfig.c (revision 20953) +++ /branches/bills_081204/ippTools/src/warptoolConfig.c (revision 20954) @@ -103,5 +103,4 @@ psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-magiced", 0, "has this exposure been magiced", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -116,5 +115,4 @@ psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-end_stage", 0, "define end stage", NULL); psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-magiced", 0, "has this exposure been magiced", false); psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);