Index: /trunk/dbconfig/cam.md =================================================================== --- /trunk/dbconfig/cam.md (revision 18028) +++ /trunk/dbconfig/cam.md (revision 18029) @@ -16,6 +16,6 @@ # the camPendingExp row gets deleted so we can not put a fkey on cam_id cam_id S64 0 # Primary Key - chip_id S64 0 # Primary Key fkey(cam_id, chip_id) ref camRun(cam_id, chip_id) - uri STR 255 + uri STR 255 # fkey(cam_id) ref camRun(cam_id) + bg F32 0.0 bg_stdev F32 0.0 Index: /trunk/dbconfig/changes.txt =================================================================== --- /trunk/dbconfig/changes.txt (revision 18028) +++ /trunk/dbconfig/changes.txt (revision 18029) @@ -173,4 +173,184 @@ -- 1.1.38 +-- convert from utf8 -> latin1 for performance/space gains +SET FOREIGN_KEY_CHECKS=0; +ALTER TABLE calDB CONVERT TO CHARACTER SET latin1; +ALTER TABLE calRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE camMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE camProcessedExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE camRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE chipMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE chipProcessedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE chipRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE detInputExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE detNormalizedExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE detNormalizedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detNormalizedStatImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detProcessedExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE detProcessedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detRegisteredImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detResidExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE detResidImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE detRunSummary CONVERT TO CHARACTER SET latin1; +ALTER TABLE detStackedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE diffInputSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE diffRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE diffSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE fakeMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE fakeProcessedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE fakeRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE flatcorrExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE flatcorrRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE guidePendingExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicInputSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicNodeResult CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicSkyfileMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicTree CONVERT TO CHARACTER SET latin1; +ALTER TABLE newExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE newImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE pstampDataStore CONVERT TO CHARACTER SET latin1; +ALTER TABLE pstampJob CONVERT TO CHARACTER SET latin1; +ALTER TABLE pstampRequest CONVERT TO CHARACTER SET latin1; +ALTER TABLE pzDataStore CONVERT TO CHARACTER SET latin1; +ALTER TABLE pzDownloadExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE pzDownloadImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE rawExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE rawImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE stackInputSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE stackRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE stackSumSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE summitExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE summitImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE warpMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE warpRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE warpSkyCellMap CONVERT TO CHARACTER SET latin1; +ALTER TABLE warpSkyfile CONVERT TO CHARACTER SET latin1; +SET FOREIGN_KEY_CHECKS=1; + -- first use of fake* tables --- just drop/recreate any fake* tables prior to this point +-- drop and recreate any fake* tables created prior to this point + +DROP TABLE IF EXISTS fakeRun; +CREATE TABLE fakeRun ( + fake_id BIGINT AUTO_INCREMENT, + cam_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + reduction VARCHAR(64), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(fake_id), + KEY(cam_id), + KEY(state), + KEY(label), + KEY(expgroup), + KEY(end_stage), + INDEX(fake_id, cam_id), + FOREIGN KEY (cam_id) + REFERENCES camRun(cam_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +DROP TABLE IF EXISTS fakeProcessedImfile; +CREATE TABLE fakeProcessedImfile ( + fake_id BIGINT AUTO_INCREMENT, + exp_id BIGINT(20), + class_id VARCHAR(64), + uri VARCHAR(255), + dtime_fake FLOAT, + hostname VARCHAR(64), + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(fake_id, exp_id, class_id), + KEY(fault), + FOREIGN KEY (fake_id) + REFERENCES fakeRun(fake_id), + FOREIGN KEY(exp_id, class_id) + REFERENCES rawImfile(exp_id, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +DROP TABLE IF EXISTS fakeMask; +CREATE TABLE fakeMask ( + label VARCHAR(64), + PRIMARY KEY(label)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +-- re-normalize camProcessedExp table +-- unrelated to other changes +ALTER TABLE camProcessedExp DROP FOREIGN KEY camProcessedExp_ibfk_1; +ALTER TABLE camProcessedExp DROP COLUMN chip_id; +ALTER TABLE camProcessedExp ADD FOREIGN KEY (cam_id) REFERENCES camRun (cam_id); + +-- insert fakeRun betwen camRun and warpRun +-- populate fakeRun/fakeProcessedImfile +-- then heal the foreign key references + +CREATE TEMPORARY TABLE warpCamMap ( + warp_id BIGINT, + cam_id BIGINT, + PRIMARY KEY(warp_id), + KEY(cam_id) +) ENGINE=MEMORY; + +INSERT INTO warpCamMap SELECT warp_id, cam_id FROM warpRun; + +ALTER TABLE warpRun drop FOREIGN KEY warpRun_ibfk_1; +ALTER TABLE warpSkyCellMap DROP FOREIGN KEY warpSkyCellMap_ibfk_1; + +ALTER TABLE warpRun change COLUMN cam_id fake_id BIGINT(20) DEFAULT NULL; +ALTER TABLE warpSkyCellMap DROP COLUMN cam_id; + +SET FOREIGN_KEY_CHECKS=0; +ALTER TABLE warpRun ADD FOREIGN KEY (fake_id) REFERENCES fakeRun (fake_id); +SET FOREIGN_KEY_CHECKS=1; +ALTER TABLE warpSkyCellMap ADD FOREIGN KEY (warp_id) REFERENCES warpRun(warp_id); + +INSERT INTO fakeRun +SELECT + NULL, + cam_id, + 'stop', + workdir, + label, + reduction, + expgroup, + dvodb, + tess_id, + end_stage, + NULL +FROM camRun +JOIN warpCamMap + using(cam_id); + +INSERT INTO fakeProcessedImfile +SELECT + fake_id, + chipRun.exp_id, + class_id, + uri, + 0, + hostname, + path_base, + fault, + NULL +FROM fakeRun +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + USING(chip_id); + +UPDATE warpRun, fakeRun, warpCamMap +SET warpRun.fake_id = fakeRun.fake_id +WHERE warpRun.warp_id = warpCamMap.warp_id + AND warpCamMap.cam_id = fakeRun.fake_id; + +DROP TABLE warpCamMap; Index: /trunk/dbconfig/warp.md =================================================================== --- /trunk/dbconfig/warp.md (revision 18028) +++ /trunk/dbconfig/warp.md (revision 18029) @@ -1,3 +1,3 @@ -# $Id: warp.md,v 1.16 2008-05-23 22:52:48 price Exp $ +# $Id: warp.md,v 1.17 2008-06-09 22:55:57 jhoblitt Exp $ # @@ -11,5 +11,5 @@ warpRun METADATA warp_id S64 0 # Primary Key AUTO_INCREMENT - cam_id S64 0 # Key INDEX(warp_id, cam_id) fkey(cam_id) ref camProcessedExp(cam_id) + fake_id S64 0 # Key INDEX(warp_id, fake_id) fkey(fake_id) ref camProcessedExp(fake_id) mode STR 64 # Key state STR 64 # Key @@ -26,10 +26,7 @@ warpSkyCellMap METADATA - warp_id S64 0 # Primary Key fkey(warp_id, cam_id) ref warpRun(warp_id, cam_id) + warp_id S64 0 # Primary Key fkey(warp_id) ref warpRun(warp_id) skycell_id STR 64 # Primary Key tess_id STR 64 # Primary Key - cam_id S64 0 # Primary Key -# class is not yet consistently carried through pXtools -# class STR 64 # Primary Key class_id STR 64 # Primary Key fault S16 0 # Key Index: /trunk/ippTools/scripts/regtest.sh =================================================================== --- /trunk/ippTools/scripts/regtest.sh (revision 18028) +++ /trunk/ippTools/scripts/regtest.sh (revision 18029) @@ -33,5 +33,5 @@ $regtool -pendingexp|| exit 1 -$regtool -addprocessedexp -exp_id $exp_id1 -exp_name t10 -inst gpc -telescope ps1 -exp_tag batman.t10 -filelevel OTA -filter r -airmass 10 -ra 1 -decl 2 -exp_type bias -exp_time 0 -bg 10 -bg_stdev 1 -bg_mean_stdev 10 -alt 10 -az 10 -ccd_temp 45 -posang 10 -object dog -dateobs "2006-10-20T10:10:10Z" -label foobar $* || exit 1 +$regtool -addprocessedexp -exp_id $exp_id1 -exp_name t10 -inst gpc -telescope ps1 -exp_tag batman.t10 -filelevel OTA -filter r -airmass 10 -ra 1 -decl 2 -exp_type object -exp_time 0 -bg 10 -bg_stdev 1 -bg_mean_stdev 10 -alt 10 -az 10 -ccd_temp 45 -posang 10 -object dog -dateobs "2006-10-20T10:10:10Z" -label foobar -tess_id moldyshoe $* || exit 1 $regtool -pendingimfile || exit 1 @@ -43,3 +43,3 @@ $regtool -pendingexp|| exit 1 -$regtool -addprocessedexp -exp_id $exp_id2 -exp_name t11 -inst gpc -telescope ps1 -exp_tag batman.t11 -filelevel OTA -filter r -airmass 11 -ra 1 -decl 2 -exp_type bias -exp_time 0 -bg 11 -bg_stdev 1 -bg_mean_stdev 11 -alt 11 -az 11 -ccd_temp 45 -posang 11 -object dog -dateobs 2006-10-20T10:10:10Z -label foobar $* || exit 1 +$regtool -addprocessedexp -exp_id $exp_id2 -exp_name t11 -inst gpc -telescope ps1 -exp_tag batman.t11 -filelevel OTA -filter r -airmass 11 -ra 1 -decl 2 -exp_type object -exp_time 0 -bg 11 -bg_stdev 1 -bg_mean_stdev 11 -alt 11 -az 11 -ccd_temp 45 -posang 11 -object dog -dateobs 2006-10-20T10:10:10Z -label foobar -tess_id moldyshoe $* || exit 1 Index: /trunk/ippTools/scripts/warptest.sh =================================================================== --- /trunk/ippTools/scripts/warptest.sh (revision 18028) +++ /trunk/ippTools/scripts/warptest.sh (revision 18029) @@ -3,7 +3,8 @@ set -o verbose -camtest.sh || exit 1 +faketest.sh || exit 1 -#warptool -definerun -mode warp -workdir file:///foo || exit 1 +#warp tool -definerun -fake_id 1 -mode warp -workdir file:///foo + #warptool -addinputexp -warp_id 1 -cam_id 1 || exit 1 #warptool -addinputexp -warp_id 1 -cam_id 2 || exit 1 @@ -15,6 +16,4 @@ warptool -tooverlap -warp_id 1 || exit 1 - -exit 1 warptool -addoverlap -mapfile mapfile.txt || exit 1 Index: /trunk/ippTools/share/Makefile.am =================================================================== --- /trunk/ippTools/share/Makefile.am (revision 18028) +++ /trunk/ippTools/share/Makefile.am (revision 18029) @@ -1,7 +1,7 @@ dist_pkgdata_DATA = \ + camtool_find_chip_id.sql \ camtool_find_pendingexp.sql \ camtool_find_pendingimfile.sql \ camtool_find_processedexp.sql \ - camtool_find_chip_id.sql \ camtool_queue_chip_id.sql \ camtool_reset_faulted_runs.sql \ @@ -18,11 +18,7 @@ detselect_search.sql \ detselect_select.sql \ - dettool_tocorrectimfile.sql \ - dettool_tocorrectexp.sql \ - dettool_stop_completed_correct_runs.sql \ dettool_find_completed_runs.sql \ dettool_normalizedstat.sql \ dettool_raw.sql \ - dettool_runs.sql \ dettool_revertdetrunsummary.sql \ dettool_revertnormalizedexp.sql \ @@ -34,5 +30,9 @@ dettool_revertresidimfile.sql\ dettool_revertstacked.sql \ + dettool_runs.sql \ dettool_start_new_iteration.sql \ + dettool_stop_completed_correct_runs.sql \ + dettool_tocorrectexp.sql \ + dettool_tocorrectimfile.sql \ dettool_todetrunsummary.sql \ dettool_tonormalizedexp.sql \ @@ -44,24 +44,32 @@ dettool_tostacked.sql \ difftool_inputskyfile.sql \ + difftool_queue.sql \ + difftool_revertdiffskyfile.sql \ difftool_skyfile.sql \ difftool_todiffskyfile.sql \ - difftool_revertdiffskyfile.sql \ - difftool_queue.sql \ + faketool_completely_processed_exp.sql \ + faketool_find_camrun.sql \ + faketool_find_pendingexp.sql \ + faketool_pendingimfile.sql \ + faketool_processedimfile.sql \ + faketool_queue_cam_id.sql \ + faketool_revertprocessedimfile.sql \ + faketool_unmasked.sql \ + flatcorr_completely_processed_chiprun.sql \ flatcorr_create_tmp_chiprundone.sql \ - flatcorr_completely_processed_chiprun.sql \ flatcorr_find_completed_floatcorruns.sql \ flatcorr_find_processedimfiles.sql \ - magictool_toprocess_inputs.sql \ - magictool_toprocess_tree.sql \ - magictool_tomask.sql \ - magictool_toskyfilemask.sql \ magictool_find_complete_diffed_exposures.sql \ magictool_find_complete_warpruns.sql \ magictool_inputskyfile.sql \ + magictool_tomask.sql \ + magictool_toprocess_inputs.sql \ + magictool_toprocess_tree.sql \ + magictool_toskyfilemask.sql \ + pxadmin_create_tables.sql \ + pxadmin_drop_tables.sql \ pztool_find_completed_exp.sql \ pztool_pendingimfile.sql \ pztool_revertcopied.sql \ - pxadmin_create_tables.sql \ - pxadmin_drop_tables.sql \ regtool_pendingexp.sql \ regtool_pendingimfile.sql \ @@ -79,5 +87,5 @@ warptool_scmap.sql \ warptool_tooverlap.sql \ + warptool_revertwarped.sql \ warptool_towarped.sql \ - warptool_warped.sql \ - warptool_revertwarped.sql + warptool_warped.sql Index: /trunk/ippTools/share/camtool_find_pendingexp.sql =================================================================== --- /trunk/ippTools/share/camtool_find_pendingexp.sql (revision 18028) +++ /trunk/ippTools/share/camtool_find_pendingexp.sql (revision 18029) @@ -20,5 +20,5 @@ -- ON chipProcessedImfile.exp_id = rawExp.exp_id LEFT JOIN camProcessedExp - ON camRun.cam_id = camProcessedExp.cam_id + USING(cam_id) LEFT JOIN camMask ON camRun.label = camMask.label Index: /trunk/ippTools/share/camtool_find_pendingimfile.sql =================================================================== --- /trunk/ippTools/share/camtool_find_pendingimfile.sql (revision 18028) +++ /trunk/ippTools/share/camtool_find_pendingimfile.sql (revision 18029) @@ -34,5 +34,5 @@ ON chipProcessedImfile.exp_id = rawExp.exp_id LEFT JOIN camProcessedExp - ON camRun.cam_id = camProcessedExp.cam_id + USING(cam_id) LEFT JOIN camMask ON camRun.label = camMask.label Index: /trunk/ippTools/share/camtool_find_processedexp.sql =================================================================== --- /trunk/ippTools/share/camtool_find_processedexp.sql (revision 18028) +++ /trunk/ippTools/share/camtool_find_processedexp.sql (revision 18029) @@ -6,5 +6,7 @@ rawExp.telescope, rawExp.filelevel -FROM camProcessedExp +FROM camRun +JOIN camProcessedExp + USING(cam_id) JOIN chipRun USING(chip_id) Index: /trunk/ippTools/share/difftool_inputskyfile.sql =================================================================== --- /trunk/ippTools/share/difftool_inputskyfile.sql (revision 18028) +++ /trunk/ippTools/share/difftool_inputskyfile.sql (revision 18029) @@ -19,14 +19,20 @@ JOIN warpRun ON diffInputSkyfile.warp_id = warpRun.warp_id - JOIN camProcessedExp - ON warpRun.cam_id = camProcessedExp.cam_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) JOIN chipRun - ON camProcessedExp.chip_id = chipRun.chip_id + USING(chip_id) JOIN chipProcessedImfile - ON chipRun.chip_id = chipProcessedImfile.chip_id + USING(chip_id) JOIN rawExp - ON chipProcessedImfile.exp_id = rawExp.exp_id + ON chipRun.exp_id = rawExp.exp_id WHERE diffRun.state = 'run' + AND warpRun.state = 'stop' + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'stop' UNION SELECT @@ -49,13 +55,19 @@ JOIN warpRun ON stackInputSkyfile.warp_id = warpRun.warp_id - JOIN camProcessedExp - ON warpRun.cam_id = camProcessedExp.cam_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) JOIN chipRun - ON camProcessedExp.chip_id = chipRun.chip_id + USING(chip_id) JOIN chipProcessedImfile - ON chipRun.chip_id = chipProcessedImfile.chip_id + USING(chip_id) JOIN rawExp - ON chipProcessedImfile.exp_id = rawExp.exp_id + ON chipRun.exp_id = rawExp.exp_id WHERE diffRun.state = 'run' + AND warpRun.state = 'stop' + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'stop' ) as Foo Index: /trunk/ippTools/share/difftool_todiffskyfile.sql =================================================================== --- /trunk/ippTools/share/difftool_todiffskyfile.sql (revision 18028) +++ /trunk/ippTools/share/difftool_todiffskyfile.sql (revision 18029) @@ -14,4 +14,6 @@ JOIN warpRun ON warpRun.warp_id = warpSkyfile.warp_id +JOIN fakeRun + USING(fake_id) JOIN camRun USING(cam_id) @@ -24,3 +26,7 @@ WHERE diffRun.state = 'run' + AND warpRun.state = 'stop' + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'stop' AND diffSkyfile.diff_id IS NULL Index: /trunk/ippTools/share/faketool_find_camrun.sql =================================================================== --- /trunk/ippTools/share/faketool_find_camrun.sql (revision 18029) +++ /trunk/ippTools/share/faketool_find_camrun.sql (revision 18029) @@ -0,0 +1,36 @@ +SELECT + * +FROM + (SELECT DISTINCT + camRun.cam_id, + chipRun.*, + rawExp.camera, + rawExp.telescope, + rawExp.dateobs, + rawExp.exp_tag, + rawExp.exp_type, + rawExp.filelevel, + rawExp.filter, + rawExp.airmass, + rawExp.ra, + rawExp.decl, + rawExp.exp_time, + rawExp.sat_pixel_frac, + rawExp.bg, + rawExp.bg_stdev, + rawExp.bg_mean_stdev, + rawExp.alt, + rawExp.az, + rawExp.ccd_temp, + rawExp.posang, + rawExp.object, + rawExp.solang + FROM camRun + JOIN chipRun + using(chip_id) + JOIN rawExp + using(exp_id) + WHERE + camRun.state = 'stop' + AND chipRun.state = 'stop' +) as Foo Index: /trunk/ippTools/share/faketool_find_pendingexp.sql =================================================================== --- /trunk/ippTools/share/faketool_find_pendingexp.sql (revision 18029) +++ /trunk/ippTools/share/faketool_find_pendingexp.sql (revision 18029) @@ -0,0 +1,34 @@ +-- this query is used by both camtool -pendingexp & camtool -addprocessedexp it +-- does a little more work then is necessary for -addprocessed but it seems +-- "cleaner" to use the same query for both cases +SELECT * FROM + (SELECT + fakeRun.*, + rawExp.exp_tag, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel + FROM fakeRun + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN chipProcessedImfile + USING(chip_id) + JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id + LEFT JOIN fakeProcessedImfile + USING(fake_id) + LEFT JOIN fakeMask + ON fakeRun.label = fakeMask.label + WHERE + fakeRun.state = 'run' + AND camRun.state = 'stop' + AND chipRun.state = 'stop' + AND fakeMask.label IS NULL + AND fakeProcessedImfile.fake_id IS NULL + GROUP BY + fakeRun.fake_id + ) as Foo Index: /trunk/ippTools/share/faketool_pendingimfile.sql =================================================================== --- /trunk/ippTools/share/faketool_pendingimfile.sql (revision 18028) +++ /trunk/ippTools/share/faketool_pendingimfile.sql (revision 18029) @@ -21,5 +21,6 @@ chipProcessedImfile.n_cr, chipProcessedImfile.n_astrom, - chipProcessedImfile.path_base, + chipProcessedImfile.path_base as chip_path_base, + camProcessedExp.path_base as cam_path_base, rawExp.exp_name, rawExp.camera, @@ -28,4 +29,6 @@ FROM fakeRun JOIN camRun + USING(cam_id) + JOIN camProcessedExp USING(cam_id) JOIN chipRun Index: /trunk/ippTools/share/faketool_revertprocessedimfile.sql =================================================================== --- /trunk/ippTools/share/faketool_revertprocessedimfile.sql (revision 18029) +++ /trunk/ippTools/share/faketool_revertprocessedimfile.sql (revision 18029) @@ -0,0 +1,8 @@ +DELETE FROM fakeProcessedImfile +USING fakeProcessedImfile, fakeRun, camRun, chipRun, rawExp +WHERE + fakeProcessedImfile.fake_id = fakeRun.fake_id + AND fakeRun.cam_id = camRun.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND fakeProcessedImfile.fault != 0 Index: /trunk/ippTools/share/faketool_unmasked.sql =================================================================== --- /trunk/ippTools/share/faketool_unmasked.sql (revision 18029) +++ /trunk/ippTools/share/faketool_unmasked.sql (revision 18029) @@ -0,0 +1,11 @@ +SELECT + * +FROM + (SELECT + fakeRun.label, + count(fakeRun.fake_id) as n_fakeruns + FROM fakeRun + LEFT JOIN fakeMask + USING (label) + WHERE fakeMask.label IS NULL + GROUP BY fakeRun.label) as fakeUnmask Index: /trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /trunk/ippTools/share/pxadmin_create_tables.sql (revision 18028) +++ /trunk/ippTools/share/pxadmin_create_tables.sql (revision 18029) @@ -328,5 +328,4 @@ CREATE TABLE camProcessedExp ( cam_id BIGINT, - chip_id BIGINT, uri VARCHAR(255), bg FLOAT, @@ -356,8 +355,8 @@ path_base VARCHAR(255), fault SMALLINT NOT NULL, - PRIMARY KEY(cam_id, chip_id), - KEY(fault), - FOREIGN KEY (cam_id, chip_id) - REFERENCES camRun(cam_id, chip_id)) + PRIMARY KEY(cam_id), + KEY(fault), + FOREIGN KEY (cam_id) + REFERENCES camRun(cam_id)) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -646,8 +645,72 @@ CREATE TABLE detRegisteredImfile (det_id BIGINT, iteration INT, class_id VARCHAR(64), uri VARCHAR(255), bg DOUBLE, bg_stdev DOUBLE, bg_mean_stdev DOUBLE, user_1 DOUBLE, user_2 DOUBLE, user_3 DOUBLE, user_4 DOUBLE, user_5 DOUBLE, path_base VARCHAR(255), fault SMALLINT NOT NULL, PRIMARY KEY(det_id, iteration, class_id), KEY(fault), FOREIGN KEY (det_id, iteration) REFERENCES detRun(det_id, iteration)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE warpRun (warp_id BIGINT AUTO_INCREMENT, cam_id BIGINT, mode VARCHAR(64), state VARCHAR(64), workdir VARCHAR(255), workdir_state VARCHAR(64), label VARCHAR(64), dvodb VARCHAR(255), tess_id VARCHAR(64), end_stage VARCHAR(64), registered DATETIME, magiced TINYINT, PRIMARY KEY(warp_id), KEY(warp_id), KEY(cam_id), KEY(mode), KEY(state), KEY(workdir_state), KEY(label), KEY(end_stage), KEY(magiced), INDEX(warp_id, cam_id), FOREIGN KEY (cam_id) REFERENCES camProcessedExp(cam_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE warpSkyCellMap (warp_id BIGINT, skycell_id VARCHAR(64), tess_id VARCHAR(64), cam_id BIGINT, class_id VARCHAR(64), fault SMALLINT, PRIMARY KEY(warp_id, skycell_id, tess_id, cam_id, class_id), KEY(fault), FOREIGN KEY (warp_id, cam_id) REFERENCES warpRun(warp_id, cam_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE warpSkyfile (warp_id BIGINT, skycell_id VARCHAR(64), tess_id VARCHAR(64), uri VARCHAR(255), path_base VARCHAR(255), bg DOUBLE, bg_stdev DOUBLE, dtime_warp FLOAT, hostname VARCHAR(64), good_frac FLOAT, xmin INT, xmax INT, ymin INT, ymax INT, ignored TINYINT, fault SMALLINT, PRIMARY KEY(warp_id, skycell_id, tess_id), KEY(good_frac), KEY(ignored), KEY(fault), FOREIGN KEY (warp_id, skycell_id, tess_id) REFERENCES warpSkyCellMap(warp_id, skycell_id, tess_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE warpMask (label VARCHAR(64), PRIMARY KEY(label)) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE warpRun ( + warp_id BIGINT AUTO_INCREMENT, + fake_id BIGINT, + mode VARCHAR(64), + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + registered DATETIME, + magiced TINYINT, + PRIMARY KEY(warp_id), + KEY(warp_id), + KEY(fake_id), + KEY(mode), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(end_stage), + KEY(magiced), + INDEX(warp_id, fake_id), + FOREIGN KEY (fake_id) + REFERENCES fakeRun(fake_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpSkyCellMap ( + warp_id BIGINT, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + class_id VARCHAR(64), + fault SMALLINT, + PRIMARY KEY(warp_id, skycell_id, tess_id, class_id), + KEY(fault), + FOREIGN KEY (warp_id) + REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpSkyfile ( + warp_id BIGINT, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + uri VARCHAR(255), + path_base VARCHAR(255), + bg DOUBLE, + bg_stdev DOUBLE, + dtime_warp FLOAT, + hostname VARCHAR(64), + good_frac FLOAT, + xmin INT, + xmax INT, + ymin INT, + ymax INT, + ignored TINYINT, + fault SMALLINT, + PRIMARY KEY(warp_id, skycell_id, tess_id), + KEY(good_frac), + KEY(ignored), KEY(fault), + FOREIGN KEY (warp_id, skycell_id, tess_id) + REFERENCES warpSkyCellMap(warp_id, skycell_id, tess_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpMask ( + label VARCHAR(64), + PRIMARY KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE stackRun (stack_id BIGINT AUTO_INCREMENT, state VARCHAR(64), workdir VARCHAR(255), dvodb VARCHAR(255), registered DATETIME, skycell_id VARCHAR(64), tess_id VARCHAR(64), PRIMARY KEY(stack_id), KEY(stack_id), KEY(state), KEY(skycell_id), KEY(tess_id)) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE stackInputSkyfile (stack_id BIGINT, warp_id BIGINT, PRIMARY KEY(stack_id, warp_id), FOREIGN KEY (stack_id) REFERENCES stackRun(stack_id), FOREIGN KEY (warp_id) REFERENCES warpSkyfile(warp_id)) ENGINE=innodb DEFAULT CHARSET=latin1; Index: /trunk/ippTools/share/stacktool_find_complete_warps.sql =================================================================== --- /trunk/ippTools/share/stacktool_find_complete_warps.sql (revision 18028) +++ /trunk/ippTools/share/stacktool_find_complete_warps.sql (revision 18029) @@ -6,4 +6,6 @@ JOIN warpSkyfile USING(warp_id) + JOIN fakeRun + USING(fake_id) JOIN camRun USING(cam_id) Index: /trunk/ippTools/share/stacktool_inputskyfile.sql =================================================================== --- /trunk/ippTools/share/stacktool_inputskyfile.sql (revision 18028) +++ /trunk/ippTools/share/stacktool_inputskyfile.sql (revision 18029) @@ -11,10 +11,12 @@ JOIN warpRun ON stackInputSkyfile.warp_id = warpRun.warp_id -JOIN camProcessedExp - ON warpRun.cam_id = camProcessedExp.cam_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) JOIN chipRun - ON camProcessedExp.chip_id = chipRun.chip_id + USING(chip_id) JOIN chipProcessedImfile - ON chipRun.chip_id = chipProcessedImfile.chip_id + USING(chip_id) JOIN rawExp ON chipProcessedImfile.exp_id = rawExp.exp_id Index: /trunk/ippTools/share/warptool_exp.sql =================================================================== --- /trunk/ippTools/share/warptool_exp.sql (revision 18028) +++ /trunk/ippTools/share/warptool_exp.sql (revision 18029) @@ -1,8 +1,8 @@ SELECT - camProcessedExp.* + fakeRun.* FROM warpRun -JOIN camProcessedExp - USING(cam_id) +JOIN fakeRun + USING(fake_id) WHERE warpRun.state = 'run' - AND camProcessedExp.fault = 0 + AND fakeRun.state = 'stop' Index: /trunk/ippTools/share/warptool_imfile.sql =================================================================== --- /trunk/ippTools/share/warptool_imfile.sql (revision 18028) +++ /trunk/ippTools/share/warptool_imfile.sql (revision 18029) @@ -1,18 +1,26 @@ SELECT DISTINCT rawImfile.*, - warpRun.cam_id, + warpRun.fake_id, chipProcessedImfile.uri as chip_uri, chipProcessedImfile.path_base as chip_path_base, camProcessedExp.path_base as cam_path_base FROM warpRun +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) JOIN camProcessedExp USING(cam_id) JOIN chipRun - ON camProcessedExp.chip_id = chipRun.chip_id + ON camRun.chip_id = chipRun.chip_id JOIN chipProcessedImfile - ON camProcessedExp.chip_id = chipProcessedImfile.chip_id + ON chipRun.chip_id = chipProcessedImfile.chip_id JOIN rawImfile -- is there any reason not to refer back to rawimfiles? - ON chipProcessedImfile.exp_id = rawImfile.exp_id AND chipProcessedImfile.class_id = rawImfile.class_id + ON chipProcessedImfile.exp_id = rawImfile.exp_id + AND chipProcessedImfile.class_id = rawImfile.class_id WHERE warpRun.state = 'run' + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'stop' Index: /trunk/ippTools/share/warptool_revertwarped.sql =================================================================== --- /trunk/ippTools/share/warptool_revertwarped.sql (revision 18028) +++ /trunk/ippTools/share/warptool_revertwarped.sql (revision 18029) @@ -1,7 +1,8 @@ DELETE FROM warpSkyfile -USING warpSkyfile, warpRun, camRun, chipRun, rawExp +USING warpSkyfile, warpRun, fakeRun, camRun, chipRun, rawExp WHERE warpSkyfile.warp_id = warpRun.warp_id - AND warpRun.cam_id = camRun.cam_id + AND warpRun.fake_id = fakeRun.fake_id + AND fakeRun.cam_id = camRun.cam_id AND camRun.chip_id = chipRun.chip_id AND chipRun.exp_id = rawExp.exp_id Index: /trunk/ippTools/share/warptool_scmap.sql =================================================================== --- /trunk/ippTools/share/warptool_scmap.sql (revision 18028) +++ /trunk/ippTools/share/warptool_scmap.sql (revision 18029) @@ -7,10 +7,18 @@ JOIN warpSkyCellMap USING(warp_id) +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) JOIN camProcessedExp - ON warpSkyCellMap.cam_id = camProcessedExp.cam_id + USING(cam_id) +JOIN chipRun + USING(chip_id) JOIN chipProcessedImfile - ON camProcessedExp.chip_id = chipProcessedImfile.chip_id + ON chipRun.chip_id = chipProcessedImfile.chip_id AND warpSkyCellMap.class_id = chipProcessedImfile.class_id --- move this as an option into the code --- WHERE +WHERE -- warpRun.state = 'run' + fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'stop' Index: /trunk/ippTools/share/warptool_tooverlap.sql =================================================================== --- /trunk/ippTools/share/warptool_tooverlap.sql (revision 18028) +++ /trunk/ippTools/share/warptool_tooverlap.sql (revision 18029) @@ -1,5 +1,5 @@ SELECT warpRun.warp_id, - warpRun.cam_id, + warpRun.fake_id, warpRun.workdir, warpRun.tess_id, @@ -9,5 +9,7 @@ warpRun.magiced FROM warpRun -JOIN camProcessedExp +JOIN fakeRun + USING(fake_id) +JOIN camRun USING(cam_id) JOIN chipRun @@ -16,11 +18,12 @@ USING(exp_id) LEFT JOIN warpSkyCellMap - USING(warp_id, cam_id) + USING(warp_id) LEFT JOIN warpMask ON warpRun.label = warpMask.label WHERE warpRun.state = 'run' - AND camProcessedExp.fault = 0 + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'stop' AND warpSkyCellMap.warp_id IS NULL - AND warpSkyCellMap.cam_id IS NULL AND warpMask.label IS NULL Index: /trunk/ippTools/share/warptool_towarped.sql =================================================================== --- /trunk/ippTools/share/warptool_towarped.sql (revision 18028) +++ /trunk/ippTools/share/warptool_towarped.sql (revision 18029) @@ -3,5 +3,6 @@ warpSkyCellMap.skycell_id, warpSkyCellMap.tess_id, - warpSkyCellMap.cam_id, + warpRun.fake_id, + camRun.cam_id, rawExp.camera, warpRun.workdir @@ -9,12 +10,14 @@ JOIN warpSkyCellMap USING(warp_id) -JOIN camProcessedExp - ON warpSkyCellMap.cam_id = camProcessedExp.cam_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) JOIN chipRun - ON camProcessedExp.chip_id = chipRun.chip_id + USING(chip_id) JOIN chipProcessedImfile - ON chipRun.chip_id = chipProcessedImfile.chip_id + USING(chip_id) JOIN rawExp - ON chipProcessedImfile.exp_id = rawExp.exp_id + ON chipRun.exp_id = rawExp.exp_id LEFT JOIN warpSkyfile ON warpRun.warp_id = warpSkyfile.warp_id @@ -25,4 +28,7 @@ WHERE warpRun.state = 'run' + AND fakeRun.state = 'stop' + AND camRun.state = 'stop' + AND chipRun.state = 'stop' AND warpSkyfile.warp_id IS NULL AND warpSkyfile.skycell_id IS NULL Index: /trunk/ippTools/src/.cvsignore =================================================================== --- /trunk/ippTools/src/.cvsignore (revision 18028) +++ /trunk/ippTools/src/.cvsignore (revision 18029) @@ -23,4 +23,5 @@ difftool stacktool +faketool dettool detselect Index: /trunk/ippTools/src/Makefile.am =================================================================== --- /trunk/ippTools/src/Makefile.am (revision 18028) +++ /trunk/ippTools/src/Makefile.am (revision 18029) @@ -8,7 +8,8 @@ flatcorr \ magictool \ - pstamptool \ + pstamptool \ pxadmin \ pxinject \ + faketool \ pzgetexp \ pzgetimfiles \ @@ -28,4 +29,5 @@ pxconfig.h \ pxdata.h \ + pxfake.h \ pxio.h \ pxregister.h \ @@ -44,7 +46,8 @@ difftool.h \ flatcorr.h \ + faketool.h \ guidetool.h \ magictool.h \ - pstamptool.h \ + pstamptool.h \ pxinject.h \ pzgetexp.h \ @@ -65,4 +68,5 @@ pxdata.c \ pxerrors.c \ + pxfake.c \ pxfault.c \ pxio.c \ @@ -124,4 +128,10 @@ camtoolConfig.c +faketool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +faketool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +faketool_SOURCES = \ + faketool.c \ + faketoolConfig.c + magictool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) magictool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la Index: /trunk/ippTools/src/camtool.c =================================================================== --- /trunk/ippTools/src/camtool.c (revision 18028) +++ /trunk/ippTools/src/camtool.c (revision 18029) @@ -27,5 +27,4 @@ #include "pxtools.h" -#include "pxwarp.h" #include "pxcam.h" #include "camtool.h" @@ -599,33 +598,32 @@ psFree(output); camProcessedExpRow *row = camProcessedExpRowAlloc( - pendingRow->cam_id, - pendingRow->chip_id, - uri, - bg, - bg_stdev, - bg_mean_stdev, - bias, - bias_stdev, - fringe_0, - fringe_1, - fringe_2, - sigma_ra, - sigma_dec, - ap_resid, - ap_resid_stdev, - zp_mean, - zp_stdev, - fwhm_major, - fwhm_minor, - dtime_detrend, - dtime_photom, - dtime_astrom, - hostname, - n_stars, - n_extended, - n_cr, - n_astrom, - path_base, - code + pendingRow->cam_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + bias, + bias_stdev, + fringe_0, + fringe_1, + fringe_2, + sigma_ra, + sigma_dec, + ap_resid, + ap_resid_stdev, + zp_mean, + zp_stdev, + fwhm_major, + fwhm_minor, + dtime_detrend, + dtime_photom, + dtime_astrom, + hostname, + n_stars, + n_extended, + n_cr, + n_astrom, + path_base, + code ); @@ -649,7 +647,7 @@ } - // should we stop here or proceed on to the warp stage? // NULL for end_stage means go as far as possible - // we can't start a warp run unlesss tess_id is defined + // XXX true for fake too? + // XXX we can't start a warp run unlesss tess_id is defined if ((pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "cam")) || (pendingRow->tess_id == NULL) || psStrcasestr(pendingRow->tess_id, "NULL")) { @@ -666,8 +664,10 @@ // else continue on... - if (!pxwarpQueueByCamID(config, + if (!pxfakeQueueByCamID(config, pendingRow->cam_id, pendingRow->workdir, pendingRow->label, + pendingRow->reduction, + pendingRow->expgroup, pendingRow->dvodb, pendingRow->tess_id, @@ -678,5 +678,5 @@ psError(PS_ERR_UNKNOWN, false, "database error"); } - psError(PS_ERR_UNKNOWN, false, "failed to queue new warpRun"); + psError(PS_ERR_UNKNOWN, false, "failed to queue new fakeRun"); psFree(pendingRow); return false; Index: /trunk/ippTools/src/faketool.c =================================================================== --- /trunk/ippTools/src/faketool.c (revision 18028) +++ /trunk/ippTools/src/faketool.c (revision 18029) @@ -32,8 +32,8 @@ #include "faketool.h" -#include "camtool.h" static bool queueMode(pxConfig *config); static bool updaterunMode(pxConfig *config); +static bool pendingexpMode(pxConfig *config); static bool pendingimfileMode(pxConfig *config); static bool addprocessedimfileMode(pxConfig *config); @@ -67,4 +67,5 @@ MODECASE(FAKETOOL_MODE_QUEUE, queueMode); MODECASE(FAKETOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(FAKETOOL_MODE_PENDINGEXP, pendingexpMode); MODECASE(FAKETOOL_MODE_PENDINGIMFILE, pendingimfileMode); MODECASE(FAKETOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileMode); @@ -171,5 +172,5 @@ // find the exp_id of all the exposures that we want to queue up. - psString query = pxDataGet("faketool_find_rawexp.sql"); + psString query = pxDataGet("faketool_find_camrun.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -179,6 +180,6 @@ if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); - psStringAppend(&query, " AND %s", whereClause); + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); psFree(whereClause); } @@ -219,12 +220,12 @@ } - // loop over our list of exp_ids + // loop over our list of cam_ids for (long i = 0; i < psArrayLength(output); i++) { psMetadata *md = output->data[i]; bool status; - psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id"); + psS64 cam_id = psMetadataLookupS64(&status, md, "cam_id"); if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_id"); + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for cam_id"); psFree(output); return false; @@ -232,10 +233,10 @@ // queue the exp - if (!pxfakeQueueByExpTag(config, exp_id, workdir, label, reduction, expgroup, dvodb, tess_id, end_stage)) { + if (!pxfakeQueueByCamID(config, cam_id, workdir, label, reduction, expgroup, dvodb, tess_id, end_stage)) { if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); } psError(PS_ERR_UNKNOWN, false, - "failed to trying to queue exp_id: %" PRId64, exp_id); + "failed to trying to queue cam_id: %" PRId64, cam_id); psFree(output); return false; @@ -354,4 +355,74 @@ +static bool pendingexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-fake_id", "fake_id", "=="); + PXOPT_COPY_STR(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_STR(config->args, where, "-chip_id", "chip_id", "=="); + + psString query = pxDataGet("faketool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fakePendingExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + static bool pendingimfileMode(pxConfig *config) { @@ -448,38 +519,17 @@ // fake_id, ext_tag, class_id are required - PXOPT_LOOKUP_STR(fake_id, config->args, "-fake_id", true, false); - PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", true, false); - PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(fake_id, config->args, "-fake_id", true, false); + PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); // optional - PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); - PXOPT_LOOKUP_F32(bg, config->args, "-bg", false, false); - PXOPT_LOOKUP_F32(bg_stdev, config->args, "-bg_stdev", false, false); - PXOPT_LOOKUP_F32(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); - PXOPT_LOOKUP_F32(bias, config->args, "-bias", false, false); - PXOPT_LOOKUP_F32(bias_stdev, config->args, "-bias_stdev", false, false); - PXOPT_LOOKUP_F32(fringe_0, config->args, "-fringe_0", false, false); - PXOPT_LOOKUP_F32(fringe_1, config->args, "-fringe_1", false, false); - PXOPT_LOOKUP_F32(fringe_2, config->args, "-fringe_2", false, false); - PXOPT_LOOKUP_F32(sigma_ra, config->args, "-sigma_ra", false, false); - PXOPT_LOOKUP_F32(sigma_dec, config->args, "-sigma_dec", false, false); - PXOPT_LOOKUP_F32(ap_resid, config->args, "-ap_resid", false, false); - PXOPT_LOOKUP_F32(ap_resid_stdev, config->args, "-ap_resid_stdev", false, false); - PXOPT_LOOKUP_F32(zp_mean, config->args, "-zp_mean", false, false); - PXOPT_LOOKUP_F32(zp_stdev, config->args, "-zp_stdev", false, false); - PXOPT_LOOKUP_F32(fwhm_major, config->args, "-fwhm_major", false, false); - PXOPT_LOOKUP_F32(fwhm_minor, config->args, "-fwhm_minor", false, false); - PXOPT_LOOKUP_F32(dtime_detrend, config->args, "-dtime_detrend", false, false); - PXOPT_LOOKUP_F32(dtime_photom, config->args, "-dtime_photom", false, false); - PXOPT_LOOKUP_F32(dtime_astrom, config->args, "-dtime_astrom", false, false); - PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); - PXOPT_LOOKUP_F32(n_stars, config->args, "-n_stars", false, false); - PXOPT_LOOKUP_F32(n_extended, config->args, "-n_extended", false, false); - PXOPT_LOOKUP_F32(n_cr, config->args, "-n_cr", false, false); - PXOPT_LOOKUP_F32(n_astrom, config->args, "-n_astrom", false, false); - PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + + PXOPT_LOOKUP_F32(dtime_fake, config->args, "-dtime_fake", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); // default values - PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); if (!psDBTransaction(config->dbh)) { @@ -493,30 +543,9 @@ class_id, uri, - bg, - bg_stdev, - bg_mean_stdev, - bias, - bias_stdev, - fringe_0, - fringe_1, - fringe_2, - sigma_ra, - sigma_dec, - ap_resid, - ap_resid_stdev, - zp_mean, - zp_stdev, - fwhm_major, - fwhm_minor, - dtime_detrend, - dtime_photom, - dtime_astrom, + dtime_fake, hostname, - n_stars, - n_extended, - n_cr, - n_astrom, path_base, - code + code, + NULL // epoch )) { // rollback @@ -748,8 +777,22 @@ PXOPT_LOOKUP_S16(code, config->args, "-code", true, false); - if (!pxSetFaultCode(config->dbh, "fakeProcessedImfile", config->where, code)) { + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_STR(fake_id, config->args, "-fake_id", false, false); + // convert fake_id into a psS64 + if (fake_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "fake_id", 0, "==", (psS64)atoll(fake_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item fake_id"); + psFree(where); + return false; + } + } + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + if (!pxSetFaultCode(config->dbh, "fakeProcessedImfile", where, code)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); - return false; - } + psFree(where); + return false; + } + psFree(where); return true; @@ -984,10 +1027,8 @@ // camQueueFakeID() can only be run after fakeRun.state has been set to // stop - if (!pxcamQueueByFakeID(config, + if (!pxwarpQueueByFakeID(config, fakeRun->fake_id, fakeRun->workdir, fakeRun->label, - fakeRun->reduction, - fakeRun->expgroup, fakeRun->dvodb, fakeRun->tess_id, Index: /trunk/ippTools/src/faketool.h =================================================================== --- /trunk/ippTools/src/faketool.h (revision 18028) +++ /trunk/ippTools/src/faketool.h (revision 18029) @@ -28,4 +28,5 @@ FAKETOOL_MODE_UPDATERUN, FAKETOOL_MODE_PENDINGIMFILE, + FAKETOOL_MODE_PENDINGEXP, FAKETOOL_MODE_ADDPROCESSEDIMFILE, FAKETOOL_MODE_PROCESSEDIMFILE, Index: /trunk/ippTools/src/faketoolConfig.c =================================================================== --- /trunk/ippTools/src/faketoolConfig.c (revision 18028) +++ /trunk/ippTools/src/faketoolConfig.c (revision 18029) @@ -232,4 +232,15 @@ + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, + "search by camtool ID", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-chip_id", 0, + "search by chiptool ID", NULL); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + // -pendingimfile psMetadata *pendingimfileArgs = psMetadataAlloc(); @@ -416,15 +427,18 @@ PXOPT_ADD_MODE("-queue", "create runs from raw exposures", - FAKETOOL_MODE_QUEUE, queueArgs); + FAKETOOL_MODE_QUEUE, queueArgs); PXOPT_ADD_MODE("-updaterun", "change fake run properties", - FAKETOOL_MODE_UPDATERUN, updaterunArgs); + FAKETOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingexp", "show pending exposures", + FAKETOOL_MODE_PENDINGEXP, pendingexpArgs); PXOPT_ADD_MODE("-pendingimfile", "show pending imfiles", - FAKETOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + FAKETOOL_MODE_PENDINGIMFILE, pendingimfileArgs); PXOPT_ADD_MODE("-addprocessedimfile", "add a processed imfile", - FAKETOOL_MODE_ADDPROCESSEDIMFILE,addprocessedimfileArgs); + FAKETOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); PXOPT_ADD_MODE("-processedimfile", "show processed imfiles", - FAKETOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); - PXOPT_ADD_MODE("-updateprocessedimfile", "change procesed imfile properties", FAKETOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileArgs); - PXOPT_ADD_MODE("-revertprocessedimfile", "undo a processed imfile", FAKETOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileArgs); + FAKETOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); + PXOPT_ADD_MODE("-updateprocessedimfile","change procesed imfile properties", FAKETOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileArgs); + PXOPT_ADD_MODE("-revertprocessedimfile", "undo a processed imfile", + FAKETOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileArgs); PXOPT_ADD_MODE("-block", "set a label block", FAKETOOL_MODE_BLOCK, blockArgs); PXOPT_ADD_MODE("-masked", "show blocked labels", FAKETOOL_MODE_MASKED, maskedArgs); Index: /trunk/ippTools/src/pxtools.h =================================================================== --- /trunk/ippTools/src/pxtools.h (revision 18028) +++ /trunk/ippTools/src/pxtools.h (revision 18029) @@ -28,13 +28,16 @@ #include +#include "pxconfig.h" #include "pxtoolsErrorCodes.h" + +#include "pxcam.h" +#include "pxchip.h" +#include "pxdata.h" +#include "pxfake.h" +#include "pxwarp.h" #include "pxio.h" -#include "pxdata.h" +#include "pxregister.h" #include "pxtag.h" -#include "pxregister.h" -#include "pxchip.h" -#include "pxcam.h" #include "pxtree.h" -#include "pxconfig.h" # define MAX_ROWS 10e9 Index: /trunk/ippTools/src/pxwarp.c =================================================================== --- /trunk/ippTools/src/pxwarp.c (revision 18028) +++ /trunk/ippTools/src/pxwarp.c (revision 18029) @@ -135,6 +135,6 @@ -bool pxwarpQueueByCamID(pxConfig *config, - psS64 cam_id, +bool pxwarpQueueByFakeID(pxConfig *config, + psS64 fake_id, char *workdir, char *label, @@ -148,5 +148,5 @@ if (!warpRunInsert(config->dbh, 0, // ID - cam_id, + fake_id, "warp", // mode "run", // state Index: /trunk/ippTools/src/pxwarp.h =================================================================== --- /trunk/ippTools/src/pxwarp.h (revision 18028) +++ /trunk/ippTools/src/pxwarp.h (revision 18029) @@ -30,6 +30,6 @@ bool pxwarpRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); -bool pxwarpQueueByCamID(pxConfig *config, - psS64 cam_id, +bool pxwarpQueueByFakeID(pxConfig *config, + psS64 fake_id, char *workdir, char *label, Index: /trunk/ippTools/src/warptool.c =================================================================== --- /trunk/ippTools/src/warptool.c (revision 18028) +++ /trunk/ippTools/src/warptool.c (revision 18029) @@ -111,5 +111,5 @@ // required options - PXOPT_LOOKUP_STR(cam_id, config->args, "-cam_id", true, false); + PXOPT_LOOKUP_STR(fake_id, config->args, "-fake_id", true, false); PXOPT_LOOKUP_STR(mode, config->args, "-mode", true, false); // check mode @@ -133,5 +133,5 @@ warpRunRow *warpRun = warpRunRowAlloc( 0, // ID - (psS64)atoll(cam_id), + (psS64)atoll(fake_id), mode, "run", // state @@ -176,5 +176,5 @@ PXOPT_LOOKUP_STR(warp_id, config->args, "-warp_id", false, false); - PXOPT_LOOKUP_STR(cam_id, config->args, "-cam_id", false, false); + PXOPT_LOOKUP_STR(fake_id, config->args, "-fake_id", false, false); PXOPT_LOOKUP_STR(chip_id, config->args, "-chip_id", false, false); PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", false, false); @@ -190,8 +190,8 @@ } - // convert cam_id into a psS64 - if (cam_id) { - if (!psMetadataAddS64(where, PS_LIST_TAIL, "cam_id", 0, "==", (psS64)atoll(cam_id))) { - psError(PS_ERR_UNKNOWN, false, "failed to add item cam_id"); + // convert fake_id into a psS64 + if (fake_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "fake_id", 0, "==", (psS64)atoll(fake_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item fake_id"); psFree(where); return false; @@ -543,5 +543,11 @@ } } else { - warpSkyCellMapInsert(config->dbh, (psS64)atoll(warp_id), NULL, NULL, 0, NULL, code); + warpSkyCellMapInsert(config->dbh, + (psS64)atoll(warp_id), + NULL, // skycell_id + NULL, // tess_id + NULL, // class_id + code // fault + ); } @@ -1028,5 +1034,5 @@ PXOPT_LOOKUP_STR(warp_id, config->args, "-warp_id", false, false); - PXOPT_LOOKUP_STR(cam_id, config->args, "-cam_id", false, false); + PXOPT_LOOKUP_STR(fake_id, config->args, "-fake_id", false, false); PXOPT_LOOKUP_STR(chip_id, config->args, "-chip_id", false, false); PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", false, false); @@ -1045,8 +1051,8 @@ PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); - // convert cam_id into a psS64 - if (cam_id) { - if (!psMetadataAddS64(where, PS_LIST_TAIL, "camRun.cam_id", 0, "==", (psS64)atoll(cam_id))) { - psError(PS_ERR_UNKNOWN, false, "failed to add item cam_id"); + // convert fake_id into a psS64 + if (fake_id) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "fakeRun.fake_id", 0, "==", (psS64)atoll(fake_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item fake_id"); psFree(where); return false; Index: /trunk/ippTools/src/warptoolConfig.c =================================================================== --- /trunk/ippTools/src/warptoolConfig.c (revision 18028) +++ /trunk/ippTools/src/warptoolConfig.c (revision 18029) @@ -49,5 +49,5 @@ // -definerun psMetadata *definerunArgs = psMetadataAlloc(); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-cam_id", 0, + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-fake_id", 0, "define camtool ID (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-mode", 0, @@ -72,10 +72,10 @@ // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); - // XXX need to allow multiple cam_ids + // XXX need to allow multiple fake_ids psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", NULL); - // XXX need to allow multiple cam_ids - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-cam_id", 0, - "search by cam_id", NULL); + // XXX need to allow multiple fake_ids + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-fake_id", 0, + "search by fake_id", NULL); // XXX need to allow multiple chip_ids psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-chip_id", 0, @@ -176,5 +176,5 @@ psMetadataAddStr(expArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", NULL); - psMetadataAddStr(expArgs, PS_LIST_TAIL, "-cam_id", 0, + psMetadataAddStr(expArgs, PS_LIST_TAIL, "-fake_id", 0, "search by camtool ID", NULL); psMetadataAddU64(expArgs, PS_LIST_TAIL, "-limit", 0, @@ -187,5 +187,5 @@ psMetadataAddStr(imfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", NULL); - psMetadataAddStr(imfileArgs, PS_LIST_TAIL, "-cam_id", 0, + psMetadataAddStr(imfileArgs, PS_LIST_TAIL, "-fake_id", 0, "search by camtool ID", NULL); psMetadataAddU64(imfileArgs, PS_LIST_TAIL, "-limit", 0, @@ -279,5 +279,5 @@ psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure tag", NULL); - psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-cam_id", 0, + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-fake_id", 0, "define phase 3 version of exposure tag", NULL); psMetadataAddU64(warpedArgs, PS_LIST_TAIL, "-limit", 0, @@ -288,5 +288,5 @@ // -revertwarped psMetadata *revertwarpedArgs = psMetadataAlloc(); - // XXX need to allow multiple cam_ids + // XXX need to allow multiple fake_ids psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", NULL); @@ -295,7 +295,7 @@ psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-tess_id", 0, "searcy by tessellation ID", NULL); - // XXX need to allow multiple cam_ids - psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-cam_id", 0, - "search by cam_id", NULL); + // XXX need to allow multiple fake_ids + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-fake_id", 0, + "search by fake_id", NULL); // XXX need to allow multiple chip_ids psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-chip_id", 0,