Index: /branches/eam_branches/eam_branch_20080918/dbconfig/.cvsignore =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/.cvsignore (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/.cvsignore (revision 22161) @@ -0,0 +1,2 @@ +ippdb.m4 +ippdb.mdc Index: /branches/eam_branches/eam_branch_20080918/dbconfig/Makefile =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/Makefile (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/Makefile (revision 22161) @@ -0,0 +1,26 @@ +SHELL=/bin/sh +GLUEFORGE=`which glueforge` + +all: ippdb.mdc + +ippdb.mdc : ipp.m4 *.md + m4 ipp.m4 > ippdb.mdc + +## please leave the output target as ippdb.src and +## move to ippdb by hand for cvs import +install: ippdb.mdc + rm -rf ../ippdb.src + $(GLUEFORGE) -i ippdb.mdc --output ../ippdb.src + chmod +x ../ippdb.src/autogen.sh + +src: ippdb.mdc + $(GLUEFORGE) -i ippdb.mdc --output ../ippdb + chmod +x ../ippdb/autogen.sh + +foo: src + $(MAKE) -C ../ippdb install + +build : ippdb ippdb.mdc + +clean: + @rm -f *~ Index: /branches/eam_branches/eam_branch_20080918/dbconfig/calibration.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/calibration.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/calibration.md (revision 22161) @@ -0,0 +1,15 @@ + +# table of the DVO databases which get calibrated +calDB METADATA + cal_id S64 0 # Primary Key AUTO_INCREMENT + dvodb STR 64 + state STR 64 +END + +calRun METADATA + cal_id S64 0 # Primary Key AUTO_INCREMENT + region STR 64 # success or failure + last_step STR 64 # fkey(cal_id) ref calDB(cal_id) + state STR 64 # +END + Index: /branches/eam_branches/eam_branch_20080918/dbconfig/cam.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/cam.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/cam.md (revision 22161) @@ -0,0 +1,50 @@ +camRun METADATA + cam_id S64 0 # Primary Key AUTO_INCREMENT + chip_id S64 0 # Key INDEX(cam_id, chip_id) fkey(chip_id) ref chipRun(chip_id) + state STR 64 # key + workdir STR 255 + workdir_state STR 64 # key + label STR 64 # key + reduction STR 64 + expgroup STR 64 # key + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key +END + +camProcessedExp METADATA +# the camPendingExp row gets deleted so we can not put a fkey on cam_id + cam_id S64 0 # Primary Key + uri STR 255 # fkey(cam_id) ref camRun(cam_id) + + bg F32 0.0 + bg_stdev F32 0.0 + bg_mean_stdev F32 0.0 + bias F32 0.0 + bias_stdev F32 0.0 + fringe_0 F32 0.0 + fringe_1 F32 0.0 + fringe_2 F32 0.0 + sigma_ra F32 0.0 + sigma_dec F32 0.0 + ap_resid F32 0.0 + ap_resid_stdev F32 0.0 + zp_mean F32 0.0 + zp_stdev F32 0.0 + fwhm_major F32 0.0 + fwhm_minor F32 0.0 + dtime_detrend F32 0.0 + dtime_photom F32 0.0 + dtime_astrom F32 0.0 + hostname STR 64 + n_stars S32 0 + n_extended S32 0 + n_cr S32 0 + n_astrom S32 0 + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +camMask METADATA + label STR 64 # Primary Key +END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/changes.txt =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/changes.txt (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/changes.txt (revision 22161) @@ -0,0 +1,470 @@ +-- How to manually change the database between versions via mySQL commands. +-- +-- This file is generated manually, and may not be complete. + + +-- Version 1.1.20 --> 1.1.21: + +alter table camProcessedExp drop nastro; +alter table camProcessedExp add (fwhm double, fwhm_range double, n_stars int(11), n_extended int(11), n_cr int(11), n_astrom int(11)); +alter table chipProcessedImfile add column (bias double, bias_stdev double, fringe_0 double, fringe_1 double, fringe_2 double, sigma_ra double, sigma_dec double, ap_resid double, ap_resid_stdev double, fwhm double, fwhm_range double, n_stars int(11), n_extended int(11), n_cr int(11), n_astrom int(11)); +alter table warpSkyfile add column path_base varchar(255) NULL DEFAULT NULL after uri; +alter table diffSkyfile add column path_base varchar(255) NULL DEFAULT NULL after uri; +alter table stackSumSkyfile add column path_base varchar(255) NULL DEFAULT NULL after uri; + + +-- Version 1.1.22 --> 1.1.23 +-- Adding support for reduction classes (which are used to specify recipes for parts of the pipeline). +# Adding 'reduction' to detRun +# Renaming 'recipe' in {chip,cam}{Pending,Processed}Exp to 'reduction' + +alter table detRun add column reduction varchar(64) NULL DEFAULT NULL after exp_type; +alter table chipPendingExp change column recipe reduction varchar(64); +alter table chipProcessedExp change column recipe reduction varchar(64); +alter table camPendingExp change column recipe reduction varchar(64); +alter table camProcessedExp change column recipe reduction varchar(64); + +-- Version ??? --> 1.1.29 + +alter table warpSkyfile add column ignored tinyint after good_frac; + +-- +-- The following set of changes were applied to update the gpc1 database on 2008/02/07 +-- + +alter table calDB change column catdir dvodb varchar(64); + +alter table camProcessedExp change column fwhm fwhm_major float; +alter table camProcessedExp change column fwhm_range fwhm_minor float; + +-- In order to keep the column order correct this is split up below +--alter table camProcessedExp add column ( bias float, bias_stdev float, fringe_0 float, fringe_1 float, fringe_2 float, ap_resid float, ap_resid_stdev float, dtime_detrend float, dtime_photom float, dtime_astrom float, hostname varchar(64) ); + +alter table camProcessedExp add column bias float after bg_mean_stdev; +alter table camProcessedExp add column bias_stdev float after bias; +alter table camProcessedExp add column fringe_0 float after bias_stdev; +alter table camProcessedExp add column fringe_1 float after fringe_0; +alter table camProcessedExp add column fringe_2 float after fringe_1; +alter table camProcessedExp add column ap_resid float after sigma_dec; +alter table camProcessedExp add column ap_resid_stdev float after ap_resid; +alter table camProcessedExp add column dtime_detrend float after fwhm_minor; +alter table camProcessedExp add column dtime_photom float after dtime_detrend; +alter table camProcessedExp add column dtime_astrom float after dtime_photom; +alter table camProcessedExp add column hostname varchar(64) after dtime_astrom; + + +alter table chipProcessedImfile change column fwhm fwhm_major float; +alter table chipProcessedImfile change column fwhm_range fwhm_minor float; + +-- In order to keep the column order correct this is split up below +-- alter table chipProcessedImfile add column ( zp_mean float, zp_stdev float, dtime_detrend float, dtime_photom float, dtime_astrom float, hostname varchar(64) ); + +alter table chipProcessedImfile add column zp_mean float after ap_resid_stdev; +alter table chipProcessedImfile add column zp_stdev float after zp_mean; +alter table chipProcessedImfile add column dtime_detrend float after fwhm_minor; +alter table chipProcessedImfile add column dtime_photom float after dtime_detrend; +alter table chipProcessedImfile add column dtime_astrom float after dtime_photom; +alter table chipProcessedImfile add column hostname varchar(64) after dtime_astrom; + + +-- In order to keep the column order correct this is split up below +-- alter table diffSkyfile add column ( dtime_diff float, hostname varchar(64) ); +alter table diffSkyfile add column dtime_diff float after bg_stdev; +alter table diffSkyfile add column hostname varchar(64) after dtime_diff; +alter table diffSkyfile change column good_frac good_frac float; + +alter table flatcorrRun add column filter varchar(64) after dvodb; + +alter table stackSumSkyfile change column good_frac good_frac float; + +-- alter table stackSumSkyfile add column ( dtime_stack float, hostname varchar(64) ); +alter table stackSumSkyfile add column dtime_stack float after bg_stdev; +alter table stackSumSkyfile add column hostname varchar(64) after dtime_stack; + +#alter table warpSkyfile add column ( dtime_warp float, hostname varchar(64) ); +alter table warpSkyfile add column dtime_warp float after bg_stdev; +alter table warpSkyfile add column hostname varchar(64) after dtime_warp; + +alter table warpSkyfile change column good_frac good_frac float; + +alter table newExp add column dvodb varchar(255) after reduction; +alter table newExp add column tess_id varchar(64) after dvodb; +alter table rawExp add column dvodb varchar(255) after reduction; +alter table rawExp add column tess_id varchar(64) after dvodb; +alter table chipRun add column tess_id varchar(64) after dvodb; +alter table camRun add column tess_id varchar(64) after dvodb; +alter table warpRun add column tess_id varchar(64) after dvodb; + +alter table camRun add column end_stage varchar(64) after tess_id; +alter table chipRun add column end_stage varchar(64) after tess_id; +alter table warpRun add column end_stage varchar(64) after tess_id; +alter table newExp add column end_stage varchar(64) after tess_id; +alter table rawExp add column end_stage varchar(64) after tess_id; +alter table camRun add key (end_stage); +alter table chipRun add key (end_stage); +alter table warpRun add key (end_stage); +alter table newExp add key (end_stage); +alter table rawExp add key (end_stage); + +alter table warpRun add column workdir_state varchar(64) after workdir; +alter table warpRun add key (workdir_state); + +alter table pzDoneImfile add column fault smallint(6); + +-- end of changes to 1.1.29 + +alter table newExp add column label varchar(64) after end_stage; + +-- 1.1.30 + +drop table pzDoneExp; +drop table pzDoneImfile; +drop table pzPendingExp; +drop table pzPendingImfile; +alter table flatcorrRun add column region VARCHAR(255) after stats; + +-- 1.1.31 + +ALTER TABLE warpRun ADD COLUMN cam_id BIGINT AFTER warp_id; +ALTER TABLE warpRun ADD KEY(cam_id); +ALTER TABLE warpRun ADD KEY(warp_id, cam_id); +ALTER TABLE warpRun ADD CONSTRAINT FOREIGN KEY(cam_id) REFERENCES +camRun(cam_id); +ALTER TABLE warpRun ADD COLUMN magiced TINYINT AFTER registered; +ALTER TABLE warpSkyCellMap DROP FOREIGN KEY warpSkyCellMap_ibfk_1; +UPDATE warpRun JOIN warpInputExp USING(warp_id) SET warpRun.cam_id = +warpInputExp.cam_id WHERE warpRun.warp_id = warpInputExp.warp_id; +ALTER TABLE warpSkyCellMap ADD CONSTRAINT FOREIGN KEY(warp_id, cam_id) +REFERENCES warpRun(warp_id, cam_id); +DROP TABLE warpInputExp; + +-- 1.1.32 +ALTER TABLE warpRun ADD COLUMN label VARCHAR(64) AFTER workdir_state; +ALTER TABLE warpRun ADD KEY(label); +CREATE TABLE warpMask (label VARCHAR(64), PRIMARY KEY(label)) ENGINE=innodb; + +-- 1.1.33 +ALTER TABLE summitExp ADD COLUMN fault smallint(6) NOT NULL AFTER imfiles; +ALTER TABLE summitExp ADD KEY(fault); + +-- 1.1.34 +ALTER TABLE rawImfile ADD KEY(exp_name); + +-- 1.1.35 +ALTER TABLE rawImfile ADD INDEX UNQIUE(exp_id, tmp_class_id); + +-- 1.1.36 +ALTER TABLE warpSkyfile ADD COLUMN (xmin INT, xmax INT, ymin INT, ymax INT); +ALTER TABLE diffSkyfile ADD COLUMN (stamps_num INT, stamps_rms FLOAT, sources INT); +ALTER TABLE rawImfile ADD COLUMN hostname VARCHAR(64) AFTER object; +ALTER TABLE rawExp ADD COLUMN hostname VARCHAR(64) AFTER object; + +-- 1.1.37 +ALTER TABLE pzDataStore ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER uri; +ALTER TABLE summitExp ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER fault; +ALTER TABLE summitImfile ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER uri; +ALTER TABLE pzDownloadExp ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER state; +ALTER TABLE pzDownloadImfile ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER fault; +ALTER TABLE newExp ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER label; +ALTER TABLE newImfile ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER uri; +ALTER TABLE rawExp ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER fault; +ALTER TABLE rawImfile ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER fault; + +-- 1.1.38 + +-- convert from utf8 -> latin1 for performance/space gains +SET FOREIGN_KEY_CHECKS=0; +ALTER TABLE calDB CONVERT TO CHARACTER SET latin1; +ALTER TABLE calRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE camMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE camProcessedExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE camRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE chipMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE chipProcessedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE chipRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE detInputExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE detNormalizedExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE detNormalizedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detNormalizedStatImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detProcessedExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE detProcessedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detRegisteredImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detResidExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE detResidImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE detRunSummary CONVERT TO CHARACTER SET latin1; +ALTER TABLE detStackedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE diffInputSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE diffRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE diffSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE fakeMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE fakeProcessedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE fakeRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE flatcorrExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE flatcorrRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE guidePendingExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicInputSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicNodeResult CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicSkyfileMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicTree CONVERT TO CHARACTER SET latin1; +ALTER TABLE newExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE newImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE pstampDataStore CONVERT TO CHARACTER SET latin1; +ALTER TABLE pstampJob CONVERT TO CHARACTER SET latin1; +ALTER TABLE pstampRequest CONVERT TO CHARACTER SET latin1; +ALTER TABLE pzDataStore CONVERT TO CHARACTER SET latin1; +ALTER TABLE pzDownloadExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE pzDownloadImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE rawExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE rawImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE stackInputSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE stackRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE stackSumSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE summitExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE summitImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE warpMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE warpRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE warpSkyCellMap CONVERT TO CHARACTER SET latin1; +ALTER TABLE warpSkyfile CONVERT TO CHARACTER SET latin1; +SET FOREIGN_KEY_CHECKS=1; + +-- first use of fake* tables +-- drop and recreate any fake* tables created prior to this point + +DROP TABLE IF EXISTS fakeRun; +CREATE TABLE fakeRun ( + fake_id BIGINT AUTO_INCREMENT, + cam_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + reduction VARCHAR(64), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(fake_id), + KEY(cam_id), + KEY(state), + KEY(label), + KEY(expgroup), + KEY(end_stage), + INDEX(fake_id, cam_id), + FOREIGN KEY (cam_id) + REFERENCES camRun(cam_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +DROP TABLE IF EXISTS fakeProcessedImfile; +CREATE TABLE fakeProcessedImfile ( + fake_id BIGINT AUTO_INCREMENT, + exp_id BIGINT(20), + class_id VARCHAR(64), + uri VARCHAR(255), + dtime_fake FLOAT, + hostname VARCHAR(64), + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(fake_id, exp_id, class_id), + KEY(fault), + FOREIGN KEY (fake_id) + REFERENCES fakeRun(fake_id), + FOREIGN KEY(exp_id, class_id) + REFERENCES rawImfile(exp_id, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +DROP TABLE IF EXISTS fakeMask; +CREATE TABLE fakeMask ( + label VARCHAR(64), + PRIMARY KEY(label)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +-- re-normalize camProcessedExp table +-- unrelated to other changes +ALTER TABLE camProcessedExp DROP FOREIGN KEY camProcessedExp_ibfk_1; +ALTER TABLE camProcessedExp DROP COLUMN chip_id; +ALTER TABLE camProcessedExp ADD FOREIGN KEY (cam_id) REFERENCES camRun (cam_id); + +-- insert fakeRun betwen camRun and warpRun +-- populate fakeRun/fakeProcessedImfile +-- then heal the foreign key references + +CREATE TEMPORARY TABLE warpCamMap ( + warp_id BIGINT, + cam_id BIGINT, + PRIMARY KEY(warp_id), + KEY(cam_id) +) ENGINE=MEMORY; + +INSERT INTO warpCamMap SELECT warp_id, cam_id FROM warpRun; + +ALTER TABLE warpRun drop FOREIGN KEY warpRun_ibfk_1; +ALTER TABLE warpSkyCellMap DROP FOREIGN KEY warpSkyCellMap_ibfk_1; + +ALTER TABLE warpRun change COLUMN cam_id fake_id BIGINT(20) DEFAULT NULL; +ALTER TABLE warpSkyCellMap DROP COLUMN cam_id; + +SET FOREIGN_KEY_CHECKS=0; +ALTER TABLE warpRun ADD FOREIGN KEY (fake_id) REFERENCES fakeRun (fake_id); +SET FOREIGN_KEY_CHECKS=1; +ALTER TABLE warpSkyCellMap ADD FOREIGN KEY (warp_id) REFERENCES warpRun(warp_id); + +INSERT INTO fakeRun +SELECT + NULL, + camRun.cam_id, + 'stop', + workdir, + label, + reduction, + expgroup, + dvodb, + tess_id, + end_stage, + NULL +FROM camRun +JOIN warpCamMap + using(cam_id); + +INSERT INTO fakeProcessedImfile +SELECT + fake_id, + chipRun.exp_id, + class_id, + uri, + 0, + hostname, + path_base, + fault, + NULL +FROM fakeRun +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + USING(chip_id); + +UPDATE warpRun, fakeRun, warpCamMap +SET warpRun.fake_id = fakeRun.fake_id +WHERE warpRun.warp_id = warpCamMap.warp_id + AND warpCamMap.cam_id = fakeRun.fake_id; + +DROP TABLE warpCamMap; + + +-- new values for chipRun.state + +update chipRun set state = 'new' where state = 'run'; +update chipRun set state = 'full' where state = 'stop'; + +-- from eam_branch_20080806 +-- add 'data_state' to detrend tables + +alter table detProcessedImfile add column data_state varchar(64) after path_base; +alter table detProcessedExp add column data_state varchar(64) after path_base; +alter table detStackedImfile add column data_state varchar(64) after user_5; +alter table detNormalizedStatImfile add column data_state varchar(64) after norm; +alter table detNormalizedImfile add column data_state varchar(64) after path_base; +alter table detNormalizedExp add column data_state varchar(64) after path_base; +alter table detResidImfile add column data_state varchar(64) after path_base; +alter table detResidExp add column data_state varchar(64) after path_base; +alter table detRunSummary add column data_state varchar(64) after iteration; +alter table detRegisteredImfile add column data_state varchar(64) after path_base; + +update camRun set state = 'new' where state = 'run'; +update camRun set state = 'full' where state = 'stop'; + +update fakeRun set state = 'new' where state = 'run'; +update fakeRun set state = 'full' where state = 'stop'; + +update warpRun set state = 'new' where state = 'run'; +update warpRun set state = 'full' where state = 'stop'; + +update stackRun set state = 'new' where state = 'run'; +update stackRun set state = 'full' where state = 'stop'; + +update diffRun set state = 'new' where state = 'run'; +update diffRun set state = 'full' where state = 'stop'; + +-- not sure when 'filter' was added to stackRun, but it is needed now: + +alter table stackRun add column filter varchar(64) after tess_id; + +-- changes to the flatcorr tables : no valid flatcorr tables have been created to date expect my tests +-- this sql just drops and recreates the flatcorrRun with the correct layout + +delete from flatcorrExp; +delete from flatcorrRun; + +drop table flatcorrExp; +drop table flatcorrRun; + +CREATE TABLE flatcorrRun ( + corr_id BIGINT AUTO_INCREMENT, + dvodb VARCHAR(64), + filter VARCHAR(64), + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + reduction VARCHAR(64), + region VARCHAR(64), + hostname VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(corr_id), + KEY(corr_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE flatcorrChipLink ( + corr_id BIGINT, + chip_id BIGINT, + PRIMARY KEY(corr_id, chip_id), + FOREIGN KEY (corr_id) REFERENCES flatcorrRun(corr_id), + FOREIGN KEY (chip_id) REFERENCES chipRun(chip_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE flatcorrCamLink ( + corr_id BIGINT, + chip_id BIGINT, + cam_id BIGINT, + PRIMARY KEY(corr_id, chip_id, cam_id), + FOREIGN KEY (corr_id) REFERENCES flatcorrRun(corr_id), + FOREIGN KEY (chip_id) REFERENCES chipRun(chip_id), + FOREIGN KEY (cam_id) REFERENCES camRun(cam_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- I initially defined flatcorrCamLin without the chip_id entry. this +-- command corrects for that, but should not be needed widely. +-- ALTER TABLE flatcorrCamLink ADD COLUMN chip_id BIGINT AFTER corr_id; +-- alter table flatcorrRun add column hostname varchar(64) after region; +-- alter table flatcorrRun add column fault smallint after hostname; +-- alter table flatcorrRun add column reduction varchar(64) after stats; + +-- changes to the chip tables to support cleanup and reprocessing + +ALTER TABLE chipProcessedImfile ADD COLUMN data_state VARCHAR(64) AFTER class_id; +UPDATE chipProcessedImfile SET data_state = 'full'; + +-- I have added 'label' and 'reduction' entries to stack and diff + +ALTER TABLE stackRun ADD COLUMN label VARCHAR(64) AFTER workdir; +ALTER TABLE stackRun ADD COLUMN reduction VARCHAR(64) AFTER label; + +ALTER TABLE stackSumSkyfile add column data_state varchar(64) after path_base; +UPDATE stackSumSkyfile SET data_state = 'full'; + +alter table diffRun add column label varchar(64) after workdir; +alter table diffRun add column reduction varchar(64) after label; + +ALTER TABLE diffSkyfile add column data_state varchar(64) after path_base; +UPDATE diffSkyfile SET data_state = 'full'; + +ALTER TABLE warpSkyfile add column data_state varchar(64) after path_base; +UPDATE warpSkyfile SET data_state = 'full'; + +ALTER TABLE fakeProcessedImfile add column data_state varchar(64) after path_base; +update fakeProcessedImfile set data_state ='full'; + Index: /branches/eam_branches/eam_branch_20080918/dbconfig/chip.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/chip.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/chip.md (revision 22161) @@ -0,0 +1,51 @@ +chipRun METADATA + chip_id S64 0 # Primary Key AUTO_INCREMENT + exp_id S64 64 # Key INDEX(chip_id, exp_id) fkey (exp_id) ref rawExp(exp_id) + state STR 64 # Key + workdir STR 255 + workdir_state STR 64 # Key + label STR 64 # Key + reduction STR 64 # Reduction class + expgroup STR 64 # Key + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key +END + +chipProcessedImfile METADATA + chip_id S64 0 # Primary Key fkey (chip_id, exp_id) ref chipRun(chip_id, exp_id) + exp_id S64 64 # Primary Key fkey (exp_id, class_id) ref rawImfile(exp_id, class_id) + class_id STR 64 # Primary Key + data_state STR 64 # Key + uri STR 255 + bg F32 0.0 + bg_stdev F32 0.0 + bg_mean_stdev F32 0.0 + bias F32 0.0 + bias_stdev F32 0.0 + fringe_0 F32 0.0 + fringe_1 F32 0.0 + fringe_2 F32 0.0 + sigma_ra F32 0.0 + sigma_dec F32 0.0 + ap_resid F32 0.0 + ap_resid_stdev F32 0.0 + zp_mean F32 0.0 + zp_stdev F32 0.0 + fwhm_major F32 0.0 + fwhm_minor F32 0.0 + dtime_detrend F32 0.0 + dtime_photom F32 0.0 + dtime_astrom F32 0.0 + hostname STR 64 + n_stars S32 0 + n_extended S32 0 + n_cr S32 0 + n_astrom S32 0 + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +chipMask METADATA + label STR 64 # Primary Key +END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/config.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/config.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/config.md (revision 22161) @@ -0,0 +1,5 @@ +glueforge METADATA + pkg_name STR ippdb + pkg_namespace STR ippdb + pkg_version STR 1.1.39 +END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/det.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/det.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/det.md (revision 22161) @@ -0,0 +1,276 @@ +detRun METADATA + det_id S64 0 # Primary Key AUTO_INCREMENT + iteration S32 0 # Key INDEX(det_id, iteration) + det_type STR 64 # Key + mode STR 64 # Key + state STR 64 # Key + filelevel STR 64 + workdir STR 255 # destination for output files + camera STR 64 + telescope STR 64 + exp_type STR 64 # XXX this should be dropped + reduction STR 64 # Reduction clas + filter STR 64 + airmass_min F32 0.0 + airmass_max F32 0.0 + exp_time_min F32 0.0 + exp_time_max F32 0.0 + ccd_temp_min F32 0.0 + ccd_temp_max F32 0.0 + posang_min F64 0.0 + posang_max F64 0.0 + registered TAI 0001-01-01T00:00:00Z + time_begin TAI 0001-01-01T00:00:00Z + time_end TAI 0001-01-01T00:00:00Z + use_begin TAI 0001-01-01T00:00:00Z + use_end TAI 0001-01-01T00:00:00Z + solang_min F32 0.0 + solang_max F32 0.0 + label STR 64 # key + ref_det_id S64 0 # reference for 'verify' and 'correction' analysis + ref_iter S32 0 # reference for 'verify' and 'correction' analysis +END + +detInputExp METADATA + det_id S64 0 # Primary Key fkey(det_id) ref detRun(det_id) + iteration S32 0 # Primary Key fkey(exp_id) ref rawExp(exp_id) + exp_id S64 64 # Primary Key INDEX(det_id, exp_id) + include BOOL f # INDEX(det_id, iteration) +END + +detProcessedImfile METADATA + det_id S64 0 # Primary Key fkey(det_id, exp_id) ref detInputExp(det_id, exp_id) + exp_id S64 64 # Primary Key fkey(exp_id, class_id) ref rawImfile(exp_id, class_id) + class_id STR 64 # Primary Key INDEX(det_id, class_id) + uri STR 255 # INDEX(det_id, exp_id) + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + fringe_0 F64 0.0 + fringe_1 F64 0.0 + fringe_2 F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +detProcessedExp METADATA + det_id S64 0 # Primary Key fkey(det_id, exp_id) ref detInputExp(det_id, exp_id) + exp_id S64 64 # Primary Key fkey(det_id, exp_id) ref detProcessedImfile(det_id, exp_id) + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + fringe_0 F64 0.0 + fringe_1 F64 0.0 + fringe_2 F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +# +# detStackedImfile does not depend on detProcessedExp. detProcesedExp is purely +# FYI values and can be calculated in parellel +# +detStackedImfile METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration) ref detInputExp(det_id, iteration) + iteration S32 0 # Primary Key fkey(det_id, class_id) ref detProcessedImfile(det_id, class_id) + class_id STR 64 # Primary Key + uri STR 255 + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 +# XXX does it make sense to 'clean' the stacked imfiled? + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +detNormalizedStatImfile METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration) ref detInputExp(det_id, iteration) + iteration S32 0 # Primary Key fkey(det_id, iteration, class_id) ref detStackedImfile(det_id, iteration, class_id) + class_id STR 64 # Primary Key + norm F32 0.0 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +## XXX this must match in fields with detRegisteredImfile +detNormalizedImfile METADATA + det_id S64 0 # Primary Key fkey(det_id) ref detInputExp(det_id) + iteration S32 0 # Primary Key fkey(det_id, iteration, class_id) ref detNormalizedStatImfile(det_id, iteration, class_id) + class_id STR 64 # Primary Key INDEX(det_id, iteration) + uri STR 255 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +detNormalizedExp METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration) ref detInputExp(det_id, iteration) + iteration S32 0 # Primary Key fkey(det_id, iteration) ref detNormalizedImfile(det_id, iteration) + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +#detMasterFrame METADATA +# det_id S64 0 # Primary Key +# iteration S32 0 # Primary Key +# comment STR 255 +#END +# +## drop? +#detMasterImfile METADATA +# det_id S64 0 # Primary Key +# class_id STR 64 # Primary Key +# uri STR 255 +# recipe STR 64 +#END + +detResidImfile METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration, exp_id) ref detInputExp(det_id, iteration, exp_id) + iteration S32 0 # Primary Key fkey(det_id, exp_id, class_id) ref detProcessedImfile(det_id, exp_id, class_id) + ref_det_id S64 0 # detrend master actually applied (same as above for 'master', but not for 'verify') + ref_iter S32 0 # detrend master actually applied (same as above for 'master', but not for 'verify') + exp_id S64 64 # Primary Key fkey(det_id, iteration) ref detNormalizedExp(det_id, iteration) + class_id STR 64 # Primary Key INDEX(det_id, iteration, exp_id) + uri STR 255 + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + bg_skewness F64 0.0 + bg_kurtosis F64 0.0 + bin_stdev F64 0.0 + fringe_0 F64 0.0 + fringe_1 F64 0.0 + fringe_2 F64 0.0 + fringe_resid_0 F64 0.0 + fringe_resid_1 F64 0.0 + fringe_resid_2 F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +detResidExp METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration, exp_id) ref detInputExp(det_id, iteration, exp_id) + iteration S32 0 # Primary Key fkey(det_id, iteration, exp_id) ref detResidImfile(det_id, iteration, exp_id) + exp_id S64 64 # Primary Key INDEX(det_id, iteration) + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + bg_skewness F64 0.0 + bg_kurtosis F64 0.0 + bin_stdev F64 0.0 + fringe_0 F64 0.0 + fringe_1 F64 0.0 + fringe_2 F64 0.0 + fringe_resid_0 F64 0.0 + fringe_resid_1 F64 0.0 + fringe_resid_2 F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + accept BOOL f + fault S16 0 # Key NOT NULL +END + +# XXX this probably should have been defined at the start of a new iteration, not at the end. +detRunSummary METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration) ref detInputExp(det_id, iteration) + iteration S32 0 # Primary Key fkey(det_id, iteration) ref detResidExp(det_id, iteration) + data_state STR 64 # full, goto_cleaned, cleaned, goto_full, goto_purged, purged + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + accept BOOL f + fault S16 0 # Key NOT NULL +END + +# +# Note: This table needs to stay more or less identical to detNormalizedImfile. +# It only exists as a seperate entity so they it can have different fkeys +# +detRegisteredImfile METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration) ref detRun(det_id, iteration) + iteration S32 0 # Primary Key + class_id STR 64 # Primary Key + uri STR 255 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +detCorrectedExp METADATA + det_id S64 0 # Primary Key fkey(det_id, exp_id) ref detInputExp(det_id, exp_id) + exp_id S64 64 # Primary Key fkey(exp_id, class_id) ref rawImfile(exp_id, class_id) + uri STR 255 # INDEX(det_id, exp_id) + corr_id S64 0 + corr_type STR 64 + recipe STR 64 + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +detCorrectedImfile METADATA + det_id S64 0 # Primary Key fkey(det_id, exp_id) ref detInputExp(det_id, exp_id) + exp_id S64 64 # Primary Key fkey(exp_id, class_id) ref rawImfile(exp_id, class_id) + class_id STR 64 # Primary Key INDEX(det_id, class_id) + uri STR 255 # INDEX(det_id, exp_id) + path_base STR 255 + fault S16 0 # Key NOT NULL +END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/diff.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/diff.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/diff.md (revision 22161) @@ -0,0 +1,45 @@ +# $Id: diff.md,v 1.10 2008-09-05 00:18:26 eugene Exp $ + +diffRun METADATA + diff_id S64 0 # Primary Key AUTO_INCREMENT + state STR 64 # Key + label STR 64 # Key + reduction STR 64 # Reduction class + workdir STR 255 + dvodb STR 255 + registered TAI NULL + skycell_id STR 64 # Key + tess_id STR 64 # Key +END + +# +# Diff Sky Cells Mode +# + +# only ever 2 per run - one template / one not +diffInputSkyfile METADATA + diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) + template BOOL f # Primary Key + stack_id S64 0 # fkey(stack_id) ref stackSumSkyfile(stack_id) + warp_id S64 0 # fkey(warp_id, skycell_id, tess_id) ref warpSkyfile(warp_id, skycell_id, tess_id) + skycell_id STR 64 # Key + tess_id STR 64 # Key +# either a input or a template + kind STR 64 # Key +END + +diffSkyfile METADATA + diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) + uri STR 255 + path_base STR 255 + data_state STR 64 # Key + bg F64 0.0 + bg_stdev F64 0.0 + stamps_num S32 0 + stamps_rms F32 0.0 + sources S32 0 + dtime_diff F32 0.0 + hostname STR 64 + good_frac F32 0.0 # Key + fault S16 0 # Key +END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/dimm.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/dimm.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/dimm.md (revision 22161) @@ -0,0 +1,21 @@ +# Table 13: DIMM Measurements Table +# Column Name Datatype Description +# Time date/time The time the DIMM observation was taken. +# sigmax float Raw dispersion in x. +# sigmay float Raw dispersion in y. +# FWHM float Dervied seeing full width at half maximum. +# RA float The coordinates of the measured star. +# DEC float The coordinates of the measured star. +# Exposure time float The exposure time of the DIMM observation. +# Telescope ID string source of the DIMM data + +dimm METADATA +# time DATETIME 2006-01-11T00:00:00 + sigmax F32 0.0 + sigmay F32 0.0 + fwhm F32 0.0 + ra F64 0.0 + decl F64 0.0 + expttime F32 0.0 + telescope_id STR 255 +END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/dome.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/dome.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/dome.md (revision 22161) @@ -0,0 +1,16 @@ +# Table 15: Dome Status Table +# Column Name Datatype Description +# Time date/time The time for which the dome status is valid. +# Azimuth float The azimuth of the dome. +# Open status boolean Whether the dome is open or not. +# Lights status boolean Whether lights are on in the dome or not. +# Track status boolean Whether dome is tracking telescope or not. + +dome METADATA +# time DATETIME 2006-01-11T00:00:00 + az F32 0.0 + open BOOL t + light BOOL t + # XXX is it possible for the dome slit to not track the telescope? ;) + track BOOL t +END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/fake.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/fake.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/fake.md (revision 22161) @@ -0,0 +1,30 @@ +fakeRun METADATA + fake_id S64 0 # Primary Key AUTO_INCREMENT + cam_id S64 0 # Key INDEX(fake_id, cam_id) fkey (cam_id) ref camRun(cam_id) + state STR 64 # Key + workdir STR 255 + label STR 64 # Key + reduction STR 64 # Reduction class + expgroup STR 64 # Key + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key + epoch UTC 0001-01-01T00:00:00Z +END + +fakeProcessedImfile METADATA + fake_id S64 0 # Primary Key fkey (fake_id) ref fakeRun(fake_id) + exp_id S64 64 # Primary Key fkey (exp_id, class_id) ref rawImfile(exp_id, class_id) + class_id STR 64 # Primary Key + uri STR 255 + dtime_fake F32 0.0 + hostname STR 64 + path_base STR 255 + data_state STR 64 # Key + fault S16 0 # Key NOT NULL + epoch UTC 0001-01-01T00:00:00Z +END + +fakeMask METADATA + label STR 64 # Primary Key +END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/flatcorr.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/flatcorr.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/flatcorr.md (revision 22161) @@ -0,0 +1,28 @@ + +# table of flat-field correction runs +flatcorrRun METADATA + corr_id S64 0 # Primary Key AUTO_INCREMENT + dvodb STR 64 + filter STR 64 + state STR 64 + workdir STR 255 + label STR 64 + reduction STR 64 + region STR 64 + hostname STR 64 + fault S16 0 # Key NOT NULL +END + +# table of Exposure-level data used for each flat-field corrction run +flatcorrChipLink METADATA + corr_id S64 0 # Primary Key fkey(corr_id) ref flatcorrRun(corr_id) + chip_id S64 0 # Primary Key fkey(chip_id) ref chipRun(chip_id) +END + +# table of Exposure-level data used for each flat-field corrction run +flatcorrCamLink METADATA + corr_id S64 0 # Primary Key fkey(corr_id) ref flatcorrRun(corr_id) + chip_id S64 0 # Primary Key fkey(chip_id) ref chipRun(chip_id) + cam_id S64 0 # Primary Key fkey(chip_id) ref chipRun(chip_id) +END + Index: /branches/eam_branches/eam_branch_20080918/dbconfig/guide.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/guide.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/guide.md (revision 22161) @@ -0,0 +1,5 @@ +guidePendingExp METADATA + guide_id S64 0 # Primary Key AUTO_INCREMENT + exp_id S64 64 # Key + recipe STR 64 +END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/ipp.m4 =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/ipp.m4 (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/ipp.m4 (revision 22161) @@ -0,0 +1,23 @@ +include(config.md) +dnl include(weather.md) +dnl include(skyp_transparency.md) +dnl include(skyp_absorption.md) +dnl include(skyp_emission.md) +dnl include(dimm.md) +dnl include(skyp_ir.md) +dnl include(dome.md) +dnl include(telescope.md) +dnl include(skycell.md) +include(tasks.md) +include(guide.md) +include(chip.md) +include(cam.md) +include(fake.md) +include(warp.md) +include(diff.md) +include(stack.md) +include(det.md) +include(magic.md) +include(calibration.md) +include(flatcorr.md) +include(pstamp.md) Index: /branches/eam_branches/eam_branch_20080918/dbconfig/magic.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/magic.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/magic.md (revision 22161) @@ -0,0 +1,47 @@ +# $Id: magic.md,v 1.10 2008-07-23 01:52:30 price Exp $ + +### Fault in magicRun indicates that the processing tree failed +magicRun METADATA + magic_id S64 0 # Primary Key AUTO_INCREMENT + exp_id S64 0 # Key + state STR 64 # Key + workdir STR 255 + workdir_state STR 255 # Key + label STR 64 # key + dvodb STR 255 + registered TAI NULL + fault S16 0 # Key +END + +magicInputSkyfile METADATA + magic_id S64 0 # Primary Key fkey(magic_id) ref magicRun(magic_id) + diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) + node STR 64 # +END + +magicTree METADATA + magic_id S64 0 # Key fkey(magic_id) ref magicRun(magic_id) + node STR 64 # Key INDEX(magic_id, node) + dep STR 64 # Key +END + +magicNodeResult METADATA + magic_id S64 0 # Primary Key fkey(magic_id) ref magicRun(magic_id) + node STR 64 # Primary Key fkey(magic_id, node) ref magicTree(magic_id, node) + uri STR 255 + fault S16 0 # Key +END + +magicMask METADATA + magic_id S64 0 # Primary Key fkey(magic_id) ref magicRun(magic_id) + uri STR 255 + streaks S32 0 + fault S16 0 # Key +END + +### I don't think we need this if magic produces mask descriptions. -- PAP. +#magicSkyfileMask METADATA +# magic_id S64 0 # Primary Key fkey(magic_id) ref magicRun(magic_id) +# diff_id S64 0 # Primary Key fkey(magic_id, diff_id) ref magicInputSkyfile(magic_id, diff_id) +# uri STR 255 # fkey(magic_id) ref magicMask(magic_id) +#END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/notes.txt =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/notes.txt (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/notes.txt (revision 22161) @@ -0,0 +1,41 @@ + +2007.06.08 : EAM + + * when changing the database schema: + * increment the pkg_version number on dbconfig/config.md + * increment the ippdb version number in ippTools/configure.ac (to match) + * increment the ippTools version number in ippTools/configure.ac + * build ippbd ('make src' in dbconfig) + * check in dbconfig, ippdb, and ippTools + +2007.06.04 : EAM + +I am adding additional fields which we will need for selecting the +input detrend images: + +rawExp : + user stat 1 + user stat 2 + user stat 3 + user stat 4 + user stat 5 + +detProcessedImfile, detProcessedExp: + fringe amplitude + user stat 1 + user stat 2 + user stat 3 + user stat 4 + user stat 5 + +detResidImfile, detResidExp: + binned stdev + fringe_0 + fringe_1 + fringe_2 + user stat 1 + user stat 2 + user stat 3 + user stat 4 + user stat 5 + Index: /branches/eam_branches/eam_branch_20080918/dbconfig/pstamp.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/pstamp.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/pstamp.md (revision 22161) @@ -0,0 +1,31 @@ +pstampDataStore METADATA + ds_id S64 0 # Primary Key AUTO_INCREMENT + state STR 64 + lastFileset STR 64 + outProduct STR 64 + uri STR 255 +END + +pstampRequest METADATA + req_id S64 0 # Primary Key AUTO_INCREMENT + ds_id S64 0 + state STR 64 + name STR 64 # UNIQUE + reqType STR 16 + outProduct STR 64 + uri STR 255 + fault S32 0 +END + +pstampJob METADATA + job_id S64 0 # Primary Key AUTO_INCREMENT + req_id S64 0 # Primary Key fkey(req_id) ref pstampRequest(req_id) + rownum STR 64 + state STR 64 + jobType STR 16 + fault S32 0 + uri STR 255 + exp_id S64 0 + outputBase STR 255 + args STR 511 +END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/skycell.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/skycell.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/skycell.md (revision 22161) @@ -0,0 +1,25 @@ +# $Id: skycell.md,v 1.4 2007-07-06 01:16:08 jhoblitt Exp $ + +skyCell METADATA + skycell_id STR 64 # Primary Key + tess_id STR 64 # Primary Key + ra1 F64 0.0 + decl1 F64 0.0 + ra2 F64 0.0 + decl2 F64 0.0 + ra3 F64 0.0 + decl3 F64 0.0 + ra4 F64 0.0 + decl4 F64 0.0 +END + +skyCellMap METADATA + skycell_id STR 64 # Primary Key + tess_id STR 64 # Primary Key + exp_id S64 64 # Primary Key + p3_version S32 0 # Primary Key +# class is not yet consistently carried through pXtools +# class STR 64 # Primary Key + class_id STR 64 # Primary Key +END + Index: /branches/eam_branches/eam_branch_20080918/dbconfig/skyp_absorption.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/skyp_absorption.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/skyp_absorption.md (revision 22161) @@ -0,0 +1,25 @@ +# Table 11: Skyprobe Line Absorption Table (sample entries) +# Column Name Datatype Description +# Time date/time The time the LRProbe observation was taken. +# Disperser ID string ID of the dispersing element +# Atm Component 1 float The strength of the 1st atmospheric component. +# Atm Component 2 float The strength of the 2nd atmospheric component. +# Atm Component 3 float The strength of the 3rd atmospheric component. +# Disperser ID string ID of the dispersing element +# Number of stars int Number of stars used to measure the absorptions. +# Astrometry coords The astrometry used on the LRProbe image. +# Exposure time float The exposure time of the LRProbe image. +# Sky brightness float The measured sky (surface) brightness, in physical units. + +skyp_absorption METADATA +# time DATETIME 2006-01-10T00:00:00 + disperser_id STR 255 + atmcomp1 F32 0.0 + atmcomp2 F32 0.0 + atmcomp3 F32 0.0 + nstars S32 0 + ra F64 0.0 + decl F64 0.0 + exptime F32 0.0 + sky_bright F64 0.0 +END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/skyp_emission.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/skyp_emission.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/skyp_emission.md (revision 22161) @@ -0,0 +1,20 @@ +# Table 12: Skyprobe Line Emission Table (sample entries) +# Column Name Datatype Description +# Time date/time The time the LRProbe observation was taken. +# Disperser ID string ID of the dispersing element +# Atm Component 1 float The strength of the 1st atmospheric component. +# Atm Component 2 float The strength of the 2nd atmospheric component. +# Atm Component 3 float The strength of the 3rd atmospheric component. +# Continuum float The strength of the continuum emission. +# Disperser ID string ID of the dispersing element +# Exposure time float The exposure time of the LRProbe image. + +skyp_emission METADATA +# time DATETIME 2006-01-11T00:00:00 + disperser_id STR 255 + atmcomp1 F32 0.0 + atmcomp2 F32 0.0 + atmcomp3 F32 0.0 + continuum F32 0.0 + exptime F32 0.0 +END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/skyp_ir.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/skyp_ir.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/skyp_ir.md (revision 22161) @@ -0,0 +1,18 @@ +# Table 14: Near IR Wide-field Camera Results Table +# Column Name Datatype Description +# Time date/time The time the NIR observation was taken. +# Sky brightness float The sky (surface) brightness in the NIR observation. +# Sky variance float The variance in the sky (surface) brightness. +# Astrometry coords The astrometry used on the NIR image. +# FOV X float field width +# FOV Y float field height + +skyp_ir METADATA +# time DATETIME 2006-01-11T00:00:00 + sky_bright F64 0.0 + sky_var F64 0.0 + ra F64 0.0 + decl F64 0.0 + fov_x F32 0.0 + fov_y F32 0.0 +END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/skyp_transparency.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/skyp_transparency.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/skyp_transparency.md (revision 22161) @@ -0,0 +1,21 @@ +# Table 10: SkyProbe Transparency Table (sample entries) +# Column Name Datatype Description +# Time date/time The time the SkyProbe image was taken. +# Filter string Filter used for SkyProbe image. +# Transparency float The derived transparency. +# Number of stars int The number of stars used to measure the transparency. +# Astrometry coords The astrometry used on the SkyProbe image. +# Exposure time float The exposure time of the SkyProbe image. +# Sky brightness float The measured sky (surface) brightness, counts / second + +skyp_transparency METADATA +# time DATETIME 2006-01-10T00:00:00 # Primary Key + filter STR 255 + trans F64 0.0 + nstars S32 0 +# astrom + ra F64 0.0 + decl F64 0.0 + exptime F32 0.0 + sky_bright F64 0.0 +END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/stack.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/stack.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/stack.md (revision 22161) @@ -0,0 +1,32 @@ +# $Id: stack.md,v 1.9 2008-09-05 00:18:44 eugene Exp $ + +stackRun METADATA + stack_id S64 0 # Primary Key AUTO_INCREMENT + state STR 64 # Key + workdir STR 255 + label STR 64 # Key + reduction STR 64 # Reduction class + dvodb STR 255 + registered TAI NULL + skycell_id STR 64 # Key + tess_id STR 64 # Key + filter STR 64 +END + +stackInputSkyfile METADATA + stack_id S64 0 # Primary Key fkey(stack_id) ref stackRun(stack_id) + warp_id S64 0 # Primary Key fkey(warp_id) ref warpSkyfile(warp_id) +END + +stackSumSkyfile METADATA + stack_id S64 0 # Primary Key fkey(stack_id) ref stackRun(stack_id) + uri STR 255 + path_base STR 255 + data_state STR 64 # Key + bg F64 0.0 + bg_stdev F64 0.0 + dtime_stack F32 0.0 # Key + hostname STR 64 + good_frac F32 0.0 # Key + fault S16 0 # Key +END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/tasks.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/tasks.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/tasks.md (revision 22161) @@ -0,0 +1,226 @@ +# $Id: tasks.md,v 1.154 2008-05-31 02:19:17 jhoblitt Exp $ + +# this table records all exposure ID ever seen from the summit +# exp_name == fileset + +# note that dec is a MySQL reserved word +# note that use is a MySQL reserved word +# note that exp is a MySQL reserved word + +# for use with this stored procedure to generate exp_ids under Mysql 5+ +# +#CREATE FUNCTION genTag (exp_name varchar(64)) RETURNS VARCHAR(64) +#BEGIN +# UPDATE expTagCounter SET counter = LAST_INSERT_ID(counter + 1); +# RETURN CONCAT(exp_name, '.', LAST_INSERT_ID()); +#END +# +#expTagCounter METADATA +# counter U64 0 +#END + +pzDataStore METADATA + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + uri STR 255 + epoch UTC 0001-01-01T00:00:00Z +END + +# list of source exposures -- updated as exposures are seen +# summitExp.imfiles is updated as filesets are queried default value should be +# -1 as NULL or 0 might be a valid value (empty fileset) +summitExp METADATA + exp_name STR 64 # Primary Key + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + dateobs UTC NULL + exp_type STR 64 + uri STR 255 + imfiles S32 0 + fault S16 0 # Key NOT NULL + epoch UTC 0001-01-01T00:00:00Z +END + +# class == type of file +# class_id == type set id +# list of source images -- updated as exposures/filesets are queried +summitImfile METADATA + exp_name STR 64 # Primary Key fkey(exp_name, camera, telescope) ref summitExp(exp_name, camera, telescope) + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + file_id STR 64 # Key + bytes S32 0 + md5sum STR 32 + class STR 64 # Primary Key + class_id STR 64 # Primary Key + uri STR 255 + epoch UTC 0001-01-01T00:00:00Z +END + +# list of exposures that have had their imfiles/files registered (but not +# downloaded) +pzDownloadExp METADATA + exp_name STR 64 # Primary Key fkey(exp_name, camera, telescope) ref summitExp(exp_name, camera, telescope) + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + state STR 64 # Key + epoch UTC 0001-01-01T00:00:00Z +END + +pzDownloadImfile METADATA + exp_name STR 64 # Primary Key fkey(exp_name, camera, telescope) ref pzDownloadExp(exp_name, camera, telescope) + camera STR 64 # Primary Key fkey(exp_name, camera, telescope, class, class_id) ref summitImfile(exp_name, camera, telescope, class, class_id) + telescope STR 64 # Primary Key + class STR 64 # Primary Key + class_id STR 64 # Primary Key + uri STR 255 + fault S16 0 # Key NOT NULL + epoch UTC 0001-01-01T00:00:00Z +END + +newExp METADATA + exp_id S64 0 # Primary Key AUTO_INCREMENT + tmp_exp_name STR 64 # Key + tmp_camera STR 64 # Key + tmp_telescope STR 64 # Key + state STR 64 # Key + workdir STR 255 # destination for output files + workdir_state STR 64 # key + reduction STR 64 # Reduction class + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key + label STR 64 # Key + epoch UTC 0001-01-01T00:00:00Z +END + +# class needs to be carried here so it can go into rawImfile and be normalized +# from there +newImfile METADATA + exp_id S64 64 # Primary Key fkey(exp_id) ref newExp(exp_id) + tmp_class_id STR 64 # Primary Key + uri STR 255 + epoch UTC 0001-01-01T00:00:00Z +END + +# paired with rawImfile +rawExp METADATA + exp_id S64 64 # Primary Key fkey(exp_id) ref newExp(exp_id) + exp_name STR 64 # Key + camera STR 64 + telescope STR 64 + dateobs UTC 0001-01-01T00:00:00Z + exp_tag STR 255 + exp_type STR 64 + filelevel STR 64 + workdir STR 255 # destination for output files + reduction STR 64 # Reduction class + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key + filter STR 64 + comment STR 80 + airmass F32 0.0 + ra F64 0.0 + decl F64 0.0 + exp_time F32 0.0 + sat_pixel_frac F32 0.0 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + alt F64 0.0 + az F64 0.0 + ccd_temp F32 0.0 + posang F64 0.0 + m1_x F32 0.0 + m1_y F32 0.0 + m1_z F32 0.0 + m1_tip F32 0.0 + m1_tilt F32 0.0 + m2_x F32 0.0 + m2_y F32 0.0 + m2_z F32 0.0 + m2_tip F32 0.0 + m2_tilt F32 0.0 + env_temperature F32 0.0 + env_humidity F32 0.0 + env_wind_speed F32 0.0 + env_wind_dir F32 0.0 + teltemp_m1 F32 0.0 + teltemp_m1cell F32 0.0 + teltemp_m2 F32 0.0 + teltemp_spider F32 0.0 + teltemp_truss F32 0.0 + teltemp_extra F32 0.0 + pon_time F32 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + object STR 64 + solang F32 0.0 + hostname STR 64 + fault S16 0 # Key NOT NULL + epoch UTC 0001-01-01T00:00:00Z +END + +rawImfile METADATA + exp_id S64 64 # Primary Key fkey(exp_id, tmp_class_id) ref newImfile(exp_id, tmp_class_id) + exp_name STR 64 # UINDEX(exp_id, tmp_class_id) + camera STR 64 + telescope STR 64 + dateobs UTC 0001-01-01T00:00:00Z + tmp_class_id STR 64 # Key + class_id STR 64 # Primary Key + uri STR 255 + exp_type STR 64 +# This field is used to set the per exp filelevel. Thus the values for all of +# the imfiles in an exposure need to be sanity checked to make sure that this +# value is in argeement. + filelevel STR 64 + filter STR 64 + comment STR 80 + airmass F32 0.0 + ra F64 0.0 + decl F64 0.0 + exp_time F32 0.0 + sat_pixel_frac F32 0.0 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + alt F64 0.0 + az F64 0.0 + ccd_temp F32 0.0 + posang F64 0.0 + m1_x F32 0.0 + m1_y F32 0.0 + m1_z F32 0.0 + m1_tip F32 0.0 + m1_tilt F32 0.0 + m2_x F32 0.0 + m2_y F32 0.0 + m2_z F32 0.0 + m2_tip F32 0.0 + m2_tilt F32 0.0 + env_temperature F32 0.0 + env_humidity F32 0.0 + env_wind_speed F32 0.0 + env_wind_dir F32 0.0 + teltemp_m1 F32 0.0 + teltemp_m1cell F32 0.0 + teltemp_m2 F32 0.0 + teltemp_spider F32 0.0 + teltemp_truss F32 0.0 + teltemp_extra F32 0.0 + pon_time F32 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + object STR 64 + hostname STR 64 + fault S16 0 # Key NOT NULL + epoch UTC 0001-01-01T00:00:00Z +END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/telescope.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/telescope.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/telescope.md (revision 22161) @@ -0,0 +1,18 @@ +# Table 16: Telescope Status +# Column Name Datatype Description +# Time date/time The time for which the telescope status is valid. +# Guide status enum The status of the guiding. +# Altitude float The telescope altitude. +# Azimuth float The telescope azimuth. +# RA float The telescope Right Ascension (ICRS ~ J2000). +# Dec float The telescope Declination (ICRS ~ J2000). + +telescope METADATA +# time DATETIME 2006-01-11T00:00:00 # Primary Key +# XXX there is currently no way to declare an enum - use str or int instead? + guide STR 255 + alt F32 0.0 + az F32 0.0 + ra F64 0.0 + decl F64 0.0 +END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/warp.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/warp.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/warp.md (revision 22161) @@ -0,0 +1,57 @@ +# $Id: warp.md,v 1.18 2008-09-05 00:19:01 eugene Exp $ + +# +# We have at least 3 different run types +# a) single epoch differing +# b) magic run operates on "complete" exposures +# c) multiple epoch differing/stacking operations +# + +# define a new warprun for a single skycell +warpRun METADATA + warp_id S64 0 # Primary Key AUTO_INCREMENT + fake_id S64 0 # Key INDEX(warp_id, fake_id) fkey(fake_id) ref camProcessedExp(fake_id) + mode STR 64 # Key + state STR 64 # Key + workdir STR 255 + workdir_state STR 64 # Key + label STR 64 # key + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key + registered TAI NULL +# if magic is T then look for the exp_id in the magic output tables + magiced BOOL f # Key +END + +warpSkyCellMap METADATA + warp_id S64 0 # Primary Key fkey(warp_id) ref warpRun(warp_id) + skycell_id STR 64 # Primary Key + tess_id STR 64 # Primary Key + class_id STR 64 # Primary Key + fault S16 0 # Key +END + +warpSkyfile METADATA + warp_id S64 0 # Primary Key fkey(warp_id, skycell_id, tess_id) ref warpSkyCellMap(warp_id, skycell_id, tess_id) + skycell_id STR 64 # Primary Key + tess_id STR 64 # Primary Key + uri STR 255 + path_base STR 255 + data_state STR 64 # Key + bg F64 0.0 + bg_stdev F64 0.0 + dtime_warp F32 0.0 + hostname STR 64 + good_frac F32 0.0 # Key + xmin S32 0 + xmax S32 0 + ymin S32 0 + ymax S32 0 + ignored BOOL f # Key + fault S16 0 # Key +END + +warpMask METADATA + label STR 64 # Primary Key +END Index: /branches/eam_branches/eam_branch_20080918/dbconfig/weather.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/dbconfig/weather.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/dbconfig/weather.md (revision 22161) @@ -0,0 +1,19 @@ +# Table 9: Weather Table: some sample weather points +# Column Name Datatype Description +# Time date/time The time the weather information was measured. +# Temperature 01 float The external temperature +# Temperature 02 float The temperature at top of the dome +# Temperature 03 float The temperature on the primary mirror +# Humidity float The relative humidity. +# Pressure float The (external) atmospheric pressure. + +weather METADATA +# time DATETIME 2006-01-10T00:00:00 # Primary Key + temp01 F32 0.0 + humi01 F32 0.0 + temp02 F32 0.0 + humi02 F32 0.0 + temp03 F32 0.0 + humi03 F32 0.0 + pressure F32 0.0 +END Index: /branches/eam_branches/eam_branch_20080918/ippScripts/.cvsignore =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/.cvsignore (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/.cvsignore (revision 22161) @@ -0,0 +1,6 @@ +Build +META.yml +Makefile +Makefile.PL +_build +blib Index: /branches/eam_branches/eam_branch_20080918/ippScripts/Build.PL =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/Build.PL (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/Build.PL (revision 22161) @@ -0,0 +1,81 @@ +use Module::Build; +# See perldoc Module::Build for details of how this works + +Module::Build->new( + module_name => 'ippscripts', + dist_version => '0.01', + author => 'Joshua Hoblitt ', + license => 'gpl', + create_makefile_pl => 'passthrough', + requires => { + 'Astro::FITS::CFITSIO' => 0, + 'Cache::File' => 0, + 'Carp' => 0, + 'Data::Dumper' => 0, + 'File::Basename' => 0, + 'File::Spec' => 0, + 'File::Temp' => 0, + 'Getopt::Long' => 0, + 'IPC::Cmd' => 0, + 'IPC::Run' => 0, + 'Math::Trig' => 0, + 'Pod::Usage' => 0, + 'PS::IPP::Config' => '1.01', + 'PS::IPP::Metadata::Config' => 0, + 'Statistics::Descriptive' => 0, + 'Storable' => 0, + 'Sys::Hostname' => 0, + }, + recommends => { + 'Test::Distribution' => '1.22', + }, + script_files => [qw( + scripts/ipp_datapath.pl + scripts/ipp_filename.pl + scripts/mdc2list.pl + scripts/register_imfile.pl + scripts/register_exp.pl + scripts/detrend_process_imfile.pl + scripts/detrend_process_exp.pl + scripts/detrend_stack.pl + scripts/detrend_norm_calc.pl + scripts/detrend_norm_apply.pl + scripts/detrend_norm_exp.pl + scripts/detrend_resid_imfile.pl + scripts/detrend_resid_exp.pl + scripts/detrend_reject_exp.pl + scripts/chip_imfile.pl + scripts/camera_exp.pl + scripts/fake_imfile.pl + scripts/warp_overlap.pl + scripts/warp_skycell.pl + scripts/diff_skycell.pl + scripts/stack_skycell.pl + scripts/magic_tree.pl + scripts/magic_process.pl + scripts/magic_mask.pl + scripts/ipp_cleanup.pl + scripts/ipp_inject_fileset.pl + scripts/ipp_inject_expname.pl + scripts/ipp_serial_inject.pl + scripts/ipp_serial_inject_split.pl + scripts/ipp_serial_inject_mosaic.pl + scripts/ipp_serial_register.pl + scripts/ipp_serial_detrend.pl + scripts/ipp_serial_chip.pl + scripts/ipp_serial_camera.pl + scripts/ipp_serial_warp.pl + scripts/ipp_serial_diff.pl + scripts/ipp_serial_stack.pl + scripts/ipp_serial_mops.pl + scripts/ipp_mops_translate.pl + scripts/ipp_simulation_data.pl + scripts/isp_trans.pl + scripts/ds9_cmf_regions.pl + scripts/ipp_detrend_combine.pl + scripts/summit_copy.pl + scripts/ipp_image_path.pl + )], + dist_abstract => 'Scripts for running the Pan-STARRS IPP', + dist_author => 'Paul Price ', +)->create_build_script; Index: /branches/eam_branches/eam_branch_20080918/ippScripts/Changes =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/Changes (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/Changes (revision 22161) @@ -0,0 +1,6 @@ +Revision history for Perl module ippScripts + +0.01 Thu Feb 23 13:55:57 2006 + - help scripts split out of the ippTools package + + Index: /branches/eam_branches/eam_branch_20080918/ippScripts/LICENSE =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/LICENSE (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/LICENSE (revision 22161) @@ -0,0 +1,260 @@ +The General Public License (GPL) +Version 2, June 1991 + +Copyright (C) 1989, 1991 Free Software Foundation, Inc. 675 Mass Ave, +Cambridge, MA 02139, USA. Everyone is permitted to copy and distribute +verbatim copies of this license document, but changing it is not allowed. + +Preamble + +The licenses for most software are designed to take away your freedom to share +and change it. By contrast, the GNU General Public License is intended to +guarantee your freedom to share and change free software--to make sure the +software is free for all its users. This General Public License applies to most of +the Free Software Foundation's software and to any other program whose +authors commit to using it. (Some other Free Software Foundation software is +covered by the GNU Library General Public License instead.) You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not price. Our +General Public Licenses are designed to make sure that you have the freedom +to distribute copies of free software (and charge for this service if you wish), that +you receive source code or can get it if you want it, that you can change the +software or use pieces of it in new free programs; and that you know you can do +these things. + +To protect your rights, we need to make restrictions that forbid anyone to deny +you these rights or to ask you to surrender the rights. These restrictions +translate to certain responsibilities for you if you distribute copies of the +software, or if you modify it. + +For example, if you distribute copies of such a program, whether gratis or for a +fee, you must give the recipients all the rights that you have. You must make +sure that they, too, receive or can get the source code. And you must show +them these terms so they know their rights. + +We protect your rights with two steps: (1) copyright the software, and (2) offer +you this license which gives you legal permission to copy, distribute and/or +modify the software. + +Also, for each author's protection and ours, we want to make certain that +everyone understands that there is no warranty for this free software. If the +software is modified by someone else and passed on, we want its recipients to +know that what they have is not the original, so that any problems introduced by +others will not reflect on the original authors' reputations. + +Finally, any free program is threatened constantly by software patents. We wish +to avoid the danger that redistributors of a free program will individually obtain +patent licenses, in effect making the program proprietary. To prevent this, we +have made it clear that any patent must be licensed for everyone's free use or +not licensed at all. + +The precise terms and conditions for copying, distribution and modification +follow. + +GNU GENERAL PUBLIC LICENSE +TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND +MODIFICATION + +0. This License applies to any program or other work which contains a notice +placed by the copyright holder saying it may be distributed under the terms of +this General Public License. The "Program", below, refers to any such program +or work, and a "work based on the Program" means either the Program or any +derivative work under copyright law: that is to say, a work containing the +Program or a portion of it, either verbatim or with modifications and/or translated +into another language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not covered by +this License; they are outside its scope. The act of running the Program is not +restricted, and the output from the Program is covered only if its contents +constitute a work based on the Program (independent of having been made by +running the Program). Whether that is true depends on what the Program does. + +1. You may copy and distribute verbatim copies of the Program's source code as +you receive it, in any medium, provided that you conspicuously and appropriately +publish on each copy an appropriate copyright notice and disclaimer of warranty; +keep intact all the notices that refer to this License and to the absence of any +warranty; and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and you may at +your option offer warranty protection in exchange for a fee. + +2. You may modify your copy or copies of the Program or any portion of it, thus +forming a work based on the Program, and copy and distribute such +modifications or work under the terms of Section 1 above, provided that you also +meet all of these conditions: + +a) You must cause the modified files to carry prominent notices stating that you +changed the files and the date of any change. + +b) You must cause any work that you distribute or publish, that in whole or in +part contains or is derived from the Program or any part thereof, to be licensed +as a whole at no charge to all third parties under the terms of this License. + +c) If the modified program normally reads commands interactively when run, you +must cause it, when started running for such interactive use in the most ordinary +way, to print or display an announcement including an appropriate copyright +notice and a notice that there is no warranty (or else, saying that you provide a +warranty) and that users may redistribute the program under these conditions, +and telling the user how to view a copy of this License. (Exception: if the +Program itself is interactive but does not normally print such an announcement, +your work based on the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If identifiable +sections of that work are not derived from the Program, and can be reasonably +considered independent and separate works in themselves, then this License, +and its terms, do not apply to those sections when you distribute them as +separate works. But when you distribute the same sections as part of a whole +which is a work based on the Program, the distribution of the whole must be on +the terms of this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest your rights to +work written entirely by you; rather, the intent is to exercise the right to control +the distribution of derivative or collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program with the +Program (or with a work based on the Program) on a volume of a storage or +distribution medium does not bring the other work under the scope of this +License. + +3. You may copy and distribute the Program (or a work based on it, under +Section 2) in object code or executable form under the terms of Sections 1 and 2 +above provided that you also do one of the following: + +a) Accompany it with the complete corresponding machine-readable source +code, which must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange; or, + +b) Accompany it with a written offer, valid for at least three years, to give any +third party, for a charge no more than your cost of physically performing source +distribution, a complete machine-readable copy of the corresponding source +code, to be distributed under the terms of Sections 1 and 2 above on a medium +customarily used for software interchange; or, + +c) Accompany it with the information you received as to the offer to distribute +corresponding source code. (This alternative is allowed only for noncommercial +distribution and only if you received the program in object code or executable +form with such an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for making +modifications to it. For an executable work, complete source code means all the +source code for all modules it contains, plus any associated interface definition +files, plus the scripts used to control compilation and installation of the +executable. However, as a special exception, the source code distributed need +not include anything that is normally distributed (in either source or binary form) +with the major components (compiler, kernel, and so on) of the operating system +on which the executable runs, unless that component itself accompanies the +executable. + +If distribution of executable or object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the source +code from the same place counts as distribution of the source code, even though +third parties are not compelled to copy the source along with the object code. + +4. You may not copy, modify, sublicense, or distribute the Program except as +expressly provided under this License. Any attempt otherwise to copy, modify, +sublicense or distribute the Program is void, and will automatically terminate +your rights under this License. However, parties who have received copies, or +rights, from you under this License will not have their licenses terminated so long +as such parties remain in full compliance. + +5. You are not required to accept this License, since you have not signed it. +However, nothing else grants you permission to modify or distribute the Program +or its derivative works. These actions are prohibited by law if you do not accept +this License. Therefore, by modifying or distributing the Program (or any work +based on the Program), you indicate your acceptance of this License to do so, +and all its terms and conditions for copying, distributing or modifying the +Program or works based on it. + +6. Each time you redistribute the Program (or any work based on the Program), +the recipient automatically receives a license from the original licensor to copy, +distribute or modify the Program subject to these terms and conditions. You +may not impose any further restrictions on the recipients' exercise of the rights +granted herein. You are not responsible for enforcing compliance by third parties +to this License. + +7. If, as a consequence of a court judgment or allegation of patent infringement +or for any other reason (not limited to patent issues), conditions are imposed on +you (whether by court order, agreement or otherwise) that contradict the +conditions of this License, they do not excuse you from the conditions of this +License. If you cannot distribute so as to satisfy simultaneously your obligations +under this License and any other pertinent obligations, then as a consequence +you may not distribute the Program at all. For example, if a patent license would +not permit royalty-free redistribution of the Program by all those who receive +copies directly or indirectly through you, then the only way you could satisfy +both it and this License would be to refrain entirely from distribution of the +Program. + +If any portion of this section is held invalid or unenforceable under any particular +circumstance, the balance of the section is intended to apply and the section as +a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any patents or other +property right claims or to contest validity of any such claims; this section has +the sole purpose of protecting the integrity of the free software distribution +system, which is implemented by public license practices. Many people have +made generous contributions to the wide range of software distributed through +that system in reliance on consistent application of that system; it is up to the +author/donor to decide if he or she is willing to distribute software through any +other system and a licensee cannot impose that choice. + +This section is intended to make thoroughly clear what is believed to be a +consequence of the rest of this License. + +8. If the distribution and/or use of the Program is restricted in certain countries +either by patents or by copyrighted interfaces, the original copyright holder who +places the Program under this License may add an explicit geographical +distribution limitation excluding those countries, so that distribution is permitted +only in or among countries not thus excluded. In such case, this License +incorporates the limitation as if written in the body of this License. + +9. The Free Software Foundation may publish revised and/or new versions of the +General Public License from time to time. Such new versions will be similar in +spirit to the present version, but may differ in detail to address new problems or +concerns. + +Each version is given a distinguishing version number. If the Program specifies a +version number of this License which applies to it and "any later version", you +have the option of following the terms and conditions either of that version or of +any later version published by the Free Software Foundation. If the Program does +not specify a version number of this License, you may choose any version ever +published by the Free Software Foundation. + +10. If you wish to incorporate parts of the Program into other free programs +whose distribution conditions are different, write to the author to ask for +permission. For software which is copyrighted by the Free Software Foundation, +write to the Free Software Foundation; we sometimes make exceptions for this. +Our decision will be guided by the two goals of preserving the free status of all +derivatives of our free software and of promoting the sharing and reuse of +software generally. + +NO WARRANTY + +11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS +NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE +COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM +"AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR +IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE +ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, +YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR +CORRECTION. + +12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED +TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY +WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS +PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES +ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM +(INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY +OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS +BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +END OF TERMS AND CONDITIONS Index: /branches/eam_branches/eam_branch_20080918/ippScripts/MANIFEST =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/MANIFEST (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/MANIFEST (revision 22161) @@ -0,0 +1,38 @@ +Build.PL +Changes +LICENSE +MANIFEST +README +scripts/ipp_datapath.pl +scripts/ipp_filename.pl +scripts/mdc2list.pl +scripts/register_imfile.pl +scripts/register_exp.pl +scripts/detrend_process_imfile.pl +scripts/detrend_process_exp.pl +scripts/detrend_stack.pl +scripts/detrend_norm_calc.pl +scripts/detrend_norm_apply.pl +scripts/detrend_norm_exp.pl +scripts/detrend_resid_exp.pl +scripts/detrend_resid_imfile.pl +scripts/detrend_reject_exp.pl +scripts/chip_imfile.pl +scripts/camera_exp.pl +scripts/warp_overlap.pl +scripts/warp_skycell.pl +scripts/diff_skycell.pl +scripts/stack_skycell.pl +scripts/ipp_serial_inject.pl +scripts/ipp_serial_inject_mosaic.pl +scripts/ipp_serial_register.pl +scripts/ipp_serial_detrend.pl +scripts/ipp_serial_chip.pl +scripts/ipp_serial_camera.pl +scripts/ipp_serial_warp.pl +scripts/ipp_serial_diff.pl +scripts/ipp_serial_stack.pl +scripts/ipp_simulation_data.pl +scripts/isp_trans.pl +scripts/ds9_cmf_regions.pl +t/00_distribution.t Index: /branches/eam_branches/eam_branch_20080918/ippScripts/README =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/README (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/README (revision 22161) @@ -0,0 +1,15 @@ +pod2text foo.pm > README + +If this is still here it means the programmer was too lazy to create the readme file. + +You can create it now by using the command shown above from this directory. + +At the very least you should be able to use this set of instructions +to install the module... + +perl Build.PL +./Build +./Build test +./Build install + +If you are on a windows box you should use 'nmake' rather than 'make'. Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/calibrate_dvo.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/calibrate_dvo.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/calibrate_dvo.pl (revision 22161) @@ -0,0 +1,196 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Storable qw(freeze thaw); +use File::Basename qw( basename); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($cal_id, $dvodb, $region, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'cal_id|i=s' => \$cal_id, + 'dvodb|c=s' => \$dvodb, + 'region|r=s' => \$region, + 'dbname|d=s' => \$dbname,# Database name + 'workdir|w=s' => \$workdir, # Working directory for output files + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --cal_id --dvodb --region", + -exitval => 3) unless + defined $cal_id and + defined $dvodb and + defined $region; + +# Look for programs we need +my $missing_tools; +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $uniphot = can_run('uniphot') or (warn "Can't find uniphot" and $missing_tools = 1); +my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); +my $caltool = can_run('caltool') or (warn "Can't find caltool" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# select the primary filters from DVO query? +my (@filters) = `photcodeList -average`; + +# parse the region (RAs,RAe:DECs,DECe) : item = +/-NNN.NNNN +my @coords = split (":", $region); +my ($RAs, $RAe) = split (",", $coords[0]); +my ($DECs, $DECe) = split (",", $coords[1]); + +# Run addstar -resort +{ + my $command = "$addstar -resort"; + $command .= "-D CATDIR $dvodb"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RESORT", $status, $dbname); + } +} + +# Run relphot (filter) for each filter +{ + foreach my $filter (@filters) { + my $command = "$relphot $filter"; + $command .= "-D CATDIR $dvodb"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RELPHOT", $status, $dbname); + } + } +} + +# Run uniphot (filter) for each filter +# XXX skip this one? run less frequently? +if (0) { + foreach my $filter (@filters) { + my $command = "$uniphot $filter"; + $command .= "-D CATDIR $dvodb"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "UNIPHOT", $status, $dbname); + } + } +} + +{ + my $command = "$relastro -objects"; + $command .= "-D CATDIR $dvodb"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RELASTRO.OBJECTS", $status, $dbname); + } +} + +{ + my $command = "$relastro -images"; + $command .= "-D CATDIR $dvodb"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RELASTRO.IMAGES", $status, $dbname); + } +} + +my $command = "$caltool -addrun"; +$command .= " -cal_id $cal_id"; +$command .= " -region $region"; +$command .= " -last_step RELASTRO.IMAGES"; +$command .= " -state 0"; +$command .= " -dbname $dbname" if defined $dbname; + +# Push the results into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $cal_id = shift; + my $region = shift; + my $last_step = shift; + my $status = shift; + my $dbname = shift; + + carp($msg); + if (defined $cal_id && defined $region && defined $last_step && defined $status and not $no_update) { + my $command = "$caltool -addcalrun"; + $command .= " -cal_id $cal_id"; + $command .= " -region $region"; + $command .= " -last_step $last_step"; + $command .= " -state $status"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Pau. + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/camera_exp.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/camera_exp.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/camera_exp.pl (revision 22161) @@ -0,0 +1,406 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $exp_tag, $cam_id, $camera, $outroot, $recipe, $dbname, $reduction, $dvodb, $verbose, $no_update, + $no_op, $redirect, $save_temps, $run_state); +GetOptions( + 'exp_tag=s' => \$exp_tag, # Exposure identifier + 'cam_id=s' => \$cam_id, # Camtool identifier + 'recipe=s' => \$recipe, # Recipe to use + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'outroot|w=s' => \$outroot, # output file base name + 'reduction=s' => \$reduction, # Reduction class + 'dvodb|w=s' => \$dvodb, # output DVO database + 'run-state=s' => \$run_state, # 'new' or 'update' + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --exp_tag --cam_id --camera --outroot", + -exitval => 3, + ) unless + defined $exp_tag and + defined $cam_id and + defined $outroot and + defined $camera; + +$ipprc->define_camera($camera); + +my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); +if ($run_state eq 'update') { + $logDest .= '.update'; +} + +$ipprc->redirect_output($logDest) if $redirect; + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; + +my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1'); # Recipe to use +&my_die("Unrecognised JPEG recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe1; + +my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2'); # Recipe to use +&my_die("Unrecognised JPEG recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe2; + +my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use +&my_die("Unrecognised ADDSTAR recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe1; + +# values to extract from output metadata and the stats to calculate +# these should be coming from the psastro results +my $CHIPSTATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "bg_mean_stdev", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bias", type => "mean", flag => "-bias", dtype => "float" }, + { name => "bias_stdev", type => "rms", flag => "-bias_stdev", dtype => "float" }, + { name => "fringe_0", type => "mean", flag => "-fringe_0", dtype => "float" }, + { name => "fringe_1", type => "rms", flag => "-fringe_1", dtype => "float" }, + { name => "fringe_2", type => "stdev", flag => "-fringe_2", dtype => "float" }, +# { name => "sigma_ra", type => "rms", flag => "-sigma_ra", dtype => "float" }, +# { name => "sigma_dec", type => "rms", flag => "-sigma_dec", dtype => "float" }, + { name => "ap_resid", type => "mean", flag => "-ap_resid", dtype => "float" }, + { name => "ap_resid_stdev", type => "rms", flag => "-ap_resid_stdev", dtype => "float" }, + { name => "zp_mean", type => "mean", flag => "-zp_mean", dtype => "float" }, + { name => "zp_stdev", type => "rms", flag => "-zp_stdev", dtype => "float" }, + { name => "fwhm_major", type => "mean", flag => "-fwhm_major", dtype => "float" }, + { name => "fwhm_minor", type => "mean", flag => "-fwhm_minor", dtype => "float" }, + { name => "dtime_detrend", type => "sum", flag => "-dtime_detrend", dtype => "float" }, + { name => "dtime_photom", type => "sum", flag => "-dtime_photom", dtype => "float" }, + { name => "dtime_astrom", type => "sum", flag => "-dtime_astrom", dtype => "float" }, + { name => "n_stars", type => "sum", flag => "-n_stars", dtype => "int" }, + { name => "n_extended", type => "sum", flag => "-n_extended", dtype => "int" }, + { name => "n_cr", type => "sum", flag => "-n_cr", dtype => "int" }, + ]; +my $chipStats = PS::IPP::Metadata::Stats->new($CHIPSTATS); # Stats parser + +# values to extract from camera-level output metadata and the stats to calculate +my $CAMSTATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "CERROR", type => "rms", flag => "-sigma_ra", dtype => "float" }, + { name => "CERROR", type => "rms", flag => "-sigma_dec", dtype => "float" }, + { name => "DT_ASTR", type => "sum", flag => "-dtime_astrom", dtype => "float" }, + { name => "NASTRO", type => "sum", flag => "-n_astrom", dtype => "int" }, +# { name => "ZP??", type => "mean", flag => "-zp_mean", dtype => "float" }, +# { name => "ZP??", type => "rms", flag => "-zp_stdev", dtype => "float" }, + ]; +my $camStats = PS::IPP::Metadata::Stats->new($CAMSTATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); + +# test for addstar and psastro: +my $psastro = can_run('psastro') or (warn "Can't find psastro" and $missing_tools = 1); +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get list of component files +my $files; # Array of component files +{ + my $command = "$camtool -pendingimfile -cam_id $cam_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform camtool: $error_code", $cam_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $cam_id, $PS_EXIT_PROG_ERROR); + + # extract the stats from the metadata + unless ($chipStats->parse($metadata)) { + &my_die("Unable to find all values in statistics output.\n", $cam_id, $PS_EXIT_PROG_ERROR); + } +} + +### not needed to have such an extensive temp file name. +my ($list1File, $list1Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b1.list.XXXX", UNLINK => !$save_temps ); # For binning 1 +my ($list2File, $list2Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b2.list.XXXX", UNLINK => !$save_temps ); # For binning 2 +my ($list3File, $list3Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b3.list.XXXX", UNLINK => !$save_temps ); # For astrometry + +# XXX we perform astrometry iff photometry output exists +my $chipObjectsExist = 0; +foreach my $file (@$files) { + # use the path_base as OUTPUT root and convert the filenames with ipprc->filename: + my $class_id = $file->{class_id}; + + # If there is only one chip, we use this name for the input to addstar + # we expect the chip analysis stage to produce psphot output (cmf file) and two binned images + my $chipObjects = $ipprc->filename("PSPHOT.OUTPUT", $file->{path_base}, $class_id); + + # if any of the output chip photometry files exist, we can run psastro / addstar below + if ($ipprc->file_exists($chipObjects)) { + $chipObjectsExist = 1; + } else { + if ($verbose) { print "skipping $chipObjects (not found)\n"; } + next; + } + + print $list1File ($ipprc->filename("PPIMAGE.BIN1", $file->{path_base}, $class_id) . "\n"); + print $list2File ($ipprc->filename("PPIMAGE.BIN2", $file->{path_base}, $class_id) . "\n"); + print $list3File ($ipprc->file_resolve($chipObjects, 0) . "\n"); +} +close $list1File; +close $list2File; +close $list3File; + +# Output products +$ipprc->outroot_prepare($outroot); + +# the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs +my $jpeg1 = $ipprc->filename("PPIMAGE.JPEG1", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); +my $jpeg2 = $ipprc->filename("PPIMAGE.JPEG2", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); +my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); +my $fpaStats = $ipprc->filename("PSASTRO.STATS", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); +my $configuration = $ipprc->filename("PSASTRO.CONFIG", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); + +if ($run_state eq 'update') { + $traceDest .= '.update'; + $fpaStats .= '.update'; +} + +# convert supplied DVO database name to UNIX filename +my $dvodbReal; +if (defined $dvodb) { + $dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO + $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ); +} + + +unless ($no_op) { + + ## build the output JPEG images first so we get them even if the astrometry fails + + # Make the jpeg for binning 1 + if ($run_state eq 'new') { + my $command = "$ppImage -list $list1Name $outroot -recipe PPIMAGE $recipe1"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $cam_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg1", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($jpeg1); + } + + # Make the jpeg for binning 2 + if ($run_state eq 'new') { + my $command = "$ppImage -list $list2Name $outroot -recipe PPIMAGE $recipe2"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $cam_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg2", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($jpeg2); + } + + # only run psastro / addstar if any of the output chip astrometry files exist (should we test for successful astrometry?) + if ($chipObjectsExist) { + # run psastro on the chipObjects, producing fpaObjects + # XXX add a ppStats call which will collect the astrometry stats + my $command; + $command = "$psastro -list $list3Name $outroot"; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + + my $do_stats; + if ($run_state eq 'new') { + $command .= " -stats $fpaStats -recipe PPSTATS CHIPSTATS"; + $command .= " -dumpconfig $configuration"; + $do_stats = 1; + } elsif ($run_state eq 'update') { + $command .= " -ipprc $configuration"; + } else { + &my_die("invalid value for run-state: $run_state", $cam_id, $PS_EXIT_CONFIG_ERROR); + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform psastro: $error_code", $cam_id, $error_code); + } + # XXX do we want to give an error if astrometry fails here? + &my_die("Unable to find expected output file: $fpaObjects", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($fpaObjects); + + if ($do_stats) { + &my_die("Unable to find expected output file: $fpaStats", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($fpaObjects); + + # Get the statistics on the processed image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($fpaStats) or &my_die("Can't open statistics file $fpaStats: $!", $cam_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + # parse the statistics MDC file + my $mdcParser = PS::IPP::Metadata::Config->new(); # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents); + unless ($metadata) { + &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_PROG_ERROR); + } + + # extract the stats from the metadata + unless ($camStats->parse($metadata)) { + &my_die("Failure extracting metadata from the statistics output file.\n", $cam_id, $PS_EXIT_PROG_ERROR); + } + } + + # run addstar on the output fpaObjects (if a DVO database is defined) + if (defined $dvodbReal and ($run_state eq 'new')) { + + ## XXX the camera analysis can either save the full set of + ## detections, or just the image metadata, in the dvodb + + ## get the addstar recipe for this camera and CAMERA reduction + $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_SYS_ERROR); + } + my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_SYS_ERROR); + + ## allow the dvodb to save only images, or the full detection set + my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY'); + + # XXX this construct requires the user to have a valid .ptolemyrc + # XXX which in turn points at ippconfig/dvo.site + # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) + # XXX this needs to be converted to addstar_client... + + my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar + my $command; + $command = "$addstar -D CAMERA $camdir -update"; + $command .= " -image" if $imagesOnly; + $command .= " -D CATDIR $dvodbReal"; + + my $realFile = $ipprc->file_resolve($fpaObjects); + $command .= " $realFile"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $cam_id, $error_code); + } + } + } +} + + +my $fpaCommand = "$camtool -cam_id $cam_id"; +if ($run_state eq 'new') { + $fpaCommand .= " -addprocessedexp"; + $fpaCommand .= " -uri UNKNOWN"; + $fpaCommand .= " -path_base $outroot"; + $fpaCommand .= $chipStats->cmdflags(); + $fpaCommand .= $camStats->cmdflags(); + $fpaCommand .= " -hostname $host" if defined $host; +} else { + $fpaCommand .= " -updaterun -state full"; +} +$fpaCommand .= " -dbname $dbname" if defined $dbname; + +# Add the result into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to add result to database: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $fpaCommand\n"; +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $cam_id = shift; # Camtool identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $cam_id and not $no_update) { + my $command = "$camtool -cam_id $cam_id"; + if ($run_state eq 'new') { + $command .= " -addprocessedexp"; + $command .= " -uri UNKNOWN"; + $command .= " -code $exit_code"; + $command .= " -path_base $outroot"; + $command .= " -path_base $outroot" if defined $outroot; + } else { + $command .= " -updateprocessedexp"; + $command .= " -code $exit_code"; + } + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/chip_imfile.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/chip_imfile.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/chip_imfile.pl (revision 22161) @@ -0,0 +1,300 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ( $exp_id, $chip_id, $class_id, $uri, $camera, $outroot, $dbname, $run_state, $reduction, $threads, $verbose, + $no_update, $no_op, $redirect ); +GetOptions( + 'exp_id=s' => \$exp_id, # Exposure identifier + 'chip_id=s' => \$chip_id, # Chiptool identifier + 'class_id=s' => \$class_id, # Class identifier + 'uri|u=s' => \$uri, # Input FITS file + 'camera|c=s' => \$camera, # Camera + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction=s' => \$reduction, # Reduction class + 'run-state=s' => \$run_state, # current state of the run (new, update) + 'threads=s' => \$threads, # Number of threads to use for ppImage + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --exp_id --chip_id --class_id --uri --camera --outroot --run-state", + -exitval => 3) unless + defined $exp_id and + defined $chip_id and + defined $class_id and + defined $uri and + defined $camera and + defined $outroot and + defined $run_state; + +$ipprc->define_camera($camera); + +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +$logDest .= ".update" if $run_state eq "update"; + +$ipprc->redirect_output($logDest) if $redirect; + +# Recipes to use based on reduction class +# the CHIP recipe should not perform astrometry anymore (2008.04.08) + +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, 'CHIP'); # Recipe to use +unless ($recipe) { + &my_die("Couldn't find selected reduction for CHIP: $reduction\n", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +} + +# values to extract from output metadata and the stats to calculate +# XXX commented-out entries are not yet defined in the output files +# XXX EAM : we are removing the astrometry analysis from chip, so we +# do not bother to calculate those stats. +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "OVER_VAL", type => "mean", flag => "-bias", dtype => "float" }, + { name => "OVER_VAL", type => "stdev", flag => "-bias_stdev", dtype => "float" }, + { name => "FRINGE_0", type => "rms", flag => "-fringe_0", dtype => "float" }, + { name => "FRINGE_RESID_0", type => "rms", flag => "-fringe_1", dtype => "float" }, + { name => "FRINGE_ERR_0", type => "rms", flag => "-fringe_2", dtype => "float" }, +# { name => "CERROR", type => "rms", flag => "-sigma_ra", dtype => "float" }, +# { name => "CERROR", type => "rms", flag => "-sigma_dec", dtype => "float" }, + { name => "APMIFIT", type => "mean", flag => "-ap_resid", dtype => "float" }, + { name => "DAPMIFIT", type => "rms", flag => "-ap_resid_stdev", dtype => "float" }, +# { name => "ZP??", type => "mean", flag => "-zp_mean", dtype => "float" }, +# { name => "ZP??", type => "rms", flag => "-zp_stdev", dtype => "float" }, + { name => "FWHM_X", type => "mean", flag => "-fwhm_major", dtype => "float" }, + { name => "FWHM_Y", type => "mean", flag => "-fwhm_minor", dtype => "float" }, + { name => "DT_DET", type => "sum", flag => "-dtime_detrend", dtype => "float" }, + { name => "DT_PHOT", type => "sum", flag => "-dtime_photom", dtype => "float" }, +# { name => "DT_ASTR", type => "sum", flag => "-dtime_astrom", dtype => "float" }, + { name => "NSTARS", type => "sum", flag => "-n_stars", dtype => "int" }, +# { name => "?", type => "sum", flag => "-n_extended", dtype => "int" }, +# { name => "?", type => "sum", flag => "-n_cr", dtype => "int" }, +# { name => "NASTRO", type => "sum", flag => "-n_astrom", dtype => "int" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +&my_die("Couldn't find input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri); + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +## these names are used in ppImage, and thus may be URIs +my $outputImage = $ipprc->filename("PPIMAGE.CHIP", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputMask = $ipprc->filename("PPIMAGE.CHIP.MASK", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputWeight = $ipprc->filename("PPIMAGE.CHIP.WEIGHT", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputBin1 = $ipprc->filename("PPIMAGE.BIN1", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputBin2 = $ipprc->filename("PPIMAGE.BIN2", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputStats = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $configuration = $ipprc->filename("PPIMAGE.CONFIG", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); + +if ($run_state eq 'update') { + $outputStats .= '.update'; + $traceDest .= '.update'; +} + +# Run ppImage +unless ($no_op) { + ## XXX can we convert ppImage log and trace to use the filerules to generate consistent names + ## XXX also stats: output should be implied by $outroot + my $command; + my $do_stats; + + if ($run_state eq "new") { + $command = "$ppImage -file $uri $outroot"; + $command .= " -recipe PPIMAGE $recipe"; + $command .= " -threads $threads" if defined $threads; + $command .= " -dbname $dbname" if defined $dbname; + $command .= " -dumpconfig $configuration"; + $command .= " -tracedest $traceDest -log $logDest"; + $do_stats = 1; + } else { + $command = "$ppImage -file $uri $outroot"; + $command .= " -ipprc $configuration"; + $command .= " -threads $threads" if defined $threads; + $command .= " -dbname $dbname" if defined $dbname; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -Db PPIMAGE:PHOTOM FALSE"; + } + if ($do_stats) { + $command .= " -recipe PPSTATS CHIPSTATS"; + $command .= " -stats $outputStats"; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $exp_id, $chip_id, $class_id, $error_code); + } + + ## get the ppImage recipe for this camera and CHIP reduction + $command = "$ppConfigDump -camera $camera -dump-recipe PPIMAGE -recipe PPIMAGE $recipe -"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + + ## allow the output images to be optional, depending on the recipe / reduction class + my $outputImageExpect = metadataLookupBool($recipeData, 'CHIP.FITS'); + if ($outputImageExpect) { + &my_die("Couldn't find expected output file: $outputImage\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage); + } + + my $outputMaskExpect = metadataLookupBool($recipeData, 'CHIP.MASK.FITS'); + if ($outputMaskExpect) { + &my_die("Couldn't find expected output file: $outputMask\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + } + + my $outputWeightExpect = metadataLookupBool($recipeData, 'CHIP.WEIGHT.FITS'); + if ($outputWeightExpect) { + &my_die("Couldn't find expected output file: $outputWeight\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); + } + + &my_die("Couldn't find expected output file: $outputBin1\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1); + &my_die("Couldn't find expected output file: $outputBin2\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2); + + if ($do_stats) { + &my_die("Couldn't find expected output file: $outputStats\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); + + # Get the statistics on the processed image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + # parse the statistics MDC file + my $mdcParser = PS::IPP::Metadata::Config->new(); # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents); + unless ($metadata) { + &my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR); + } + + # extract the stats from the metadata + unless ($stats->parse($metadata)) { + &my_die("Failure extracting metadata from the statistics output file.\n", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR); + } + } +} + +my $command; +if ($run_state eq 'new') { + # command to update database + $command = "$chiptool -addprocessedimfile"; + $command .= " -exp_id $exp_id"; + $command .= " -chip_id $chip_id"; + $command .= " -class_id $class_id"; + $command .= " -uri $outputImage"; + $command .= " -path_base $outroot"; + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + $command .= $stats->cmdflags(); +} else { + $command = "$chiptool -tofullimfile"; + $command .= " -chip_id $chip_id"; + $command .= " -class_id $class_id"; + $command .= " -dbname $dbname" if defined $dbname; +} + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform chiptool -addprocessedimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_id = shift; # Chiptool identifier + my $chip_id = shift; # Chiptool identifier + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + # run_state, outputImage, and outroot are globals + + carp($msg); + if (defined $chip_id and defined $class_id and not $no_update) { + my $command; + if ($run_state eq 'new') { + $command = "$chiptool -addprocessedimfile"; + $command .= " -exp_id $exp_id"; + $command .= " -uri $outputImage"; + $command .= " -path_base $outroot"; + $command .= " -hostname $host" if defined $host; + } else { + $command .= "$chiptool -updateprocessedimfile"; + } + $command .= " -chip_id $chip_id"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_correct_imfile.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_correct_imfile.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_correct_imfile.pl (revision 22161) @@ -0,0 +1,156 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $det_id, $exp_id, $class_id, $det_type, $exp_tag, $input_uri, $camera, $dbname, $workdir, $reduction, + $verbose, $no_update, $no_op, $outroot, $redirect, $corr_id, $corr_type, $corr_uri ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'exp_id|e=s' => \$exp_id, + 'class_id|i=s' => \$class_id, + 'det_type|t=s' => \$det_type, + 'exp_tag|=s' => \$exp_tag, + 'input_uri|u=s' => \$input_uri, + 'corr_id|u=s' => \$corr_id, + 'corr_type|u=s' => \$corr_type, + 'camera|c=s' => \$camera, + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'outroot' => \$outroot, + 'redirect-output' => \$redirect, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --exp_id --class_id --det_type --exp_tag --input_uri --camera", + -exitval => 3) + unless defined $det_id + and defined $exp_id + and defined $class_id + and defined $det_type + and defined $exp_tag + and defined $input_uri + and defined $camera; + +# XXX this exits with status = 0 on failure +$ipprc->define_camera($camera); + +if ($redirect) { + die "must suplly --outroot with --redirect-output" if !defined ($outroot); + my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) + or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_CONFIG_ERROR); + $ipprc->redirect_output($logDest); +} + +# Recipes to use as a function of detrend type +$reduction = "DETREND" unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, uc($det_type) . '_PROCESS'); # Recipe name to use + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $dvocorr = can_run('dvoApplyCorr') or (warn "Can't find dvoApplyCorr" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +&my_die("Couldn't find input file: $input_uri\n", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input_uri); + +$workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir; + +# detselect -select -det_id $corr_id -class_id $class_id; + +my $outputRoot = $ipprc->file_prepare( "$exp_tag/$exp_tag.detproc.$det_id", $workdir, $input_uri ); +my $outputImage = $ipprc->filename("DVOCORR.OUTPUT", $outputRoot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + +# Run dvoApplyCorr +unless ($no_op) { + my $command = "$dvocorr -file $input_uri $outputRoot"; + $command .= " -corr $corr_uri"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $class_id, $error_code); + } + + &my_die("Couldn't find expected output file: $outputImage", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputImage); +} + +# command to update database +my $command = "$dettool -addcorrectedimfile"; +$command .= " -det_id $det_id"; +$command .= " -exp_id $exp_id"; +$command .= " -class_id $class_id"; +$command .= " -uri $outputImage"; +$command .= " -path_base $outputRoot"; +$command .= " -dbname $dbname" if defined $dbname; + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addcorrectedimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $exp_id = shift; # Exposure tag + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $exp_id and defined $class_id and not $no_update) { + my $command = "$dettool -addcorrectedimfile"; + $command .= " -det_id $det_id"; + $command .= " -exp_id $exp_id"; + $command .= " -class_id $class_id"; + $command .= " -path_base $outputRoot"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_norm_apply.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_norm_apply.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_norm_apply.pl (revision 22161) @@ -0,0 +1,234 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line +my ( $det_id, $iter, $class_id, $value, $input_uri, $camera, $det_type, $outroot, $dbname, $verbose, + $no_update, $no_op, $redirect ); +GetOptions( + 'det_id|d=s' => \$det_id, # Detrend ID + 'iteration|n=s' => \$iter, # Iteration + 'class_id|i=s' => \$class_id, # Class ID + 'value|v=s' => \$value, # Value to multiple (for normalisation) + 'input_uri|u=s' => \$input_uri, # Input file + 'camera|c=s' => \$camera, # Camera + 'det_type|t=s' => \$det_type, # Detrend type + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database + 'no-op' => \$no_op, # Don't do any operations + 'redirect-output' => \$redirect, # send output from script to LOG.IMFILE + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --class_id --value --input_uri --camera --det_type --outroot", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $class_id and + defined $value and + defined $input_uri and + defined $camera and + defined $det_type and + defined $outroot; + +$ipprc->define_camera($camera); + +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) + or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); +$ipprc->redirect_output($logDest) if $redirect; + +my $RECIPE_PPIMAGE = 'PPIMAGE_N'; # Recipe to use with ppImage + +# Define which detrend types we normalise +use constant DETTYPE => { + 'bias' => 'bias', + 'dark' => 'dark', + 'dark_premask' => 'dark', + 'shutter' => 'shutter', + 'flat_premask' => 'flat', + 'domeflat_premask' => 'flat', + 'skyflat_premask' => 'flat', + 'flat_raw' => 'flat', + 'domeflat_raw' => 'flat', + 'skyflat_raw' => 'flat', + 'flat' => 'flat', + 'domeflat' => 'flat', + 'skyflat' => 'flat', + 'fringe' => 'fringe', + 'mask' => 'mask', + 'darkmask' => 'mask', + 'flatmask' => 'mask', + }; + +# convert supplied detrend type to a controlled namespace +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR) unless exists DETTYPE()->{lc($det_type)}; +my $det_type_real = DETTYPE()->{lc($det_type)}; + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC DETTOOL FLAG + { name => "ROBUST_MEDIAN", type => "clipmean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "clipstdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +&my_die("Couldn't find input file: $input_uri\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input_uri); + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $outFile = ($det_type_real eq "mask") ? 'PPIMAGE.OUTPUT.DETMASK' : 'PPIMAGE.OUTPUT';; # XXXX something of a hack (too many places to control things...) + +my $RECIPE_PPSTATS = ($det_type_real eq "dark") ? 'DARKSTATS' : 'CHIPSTATS';; # XXXX something of a hack (too many places to control things...) + +my $output = $ipprc->filename($outFile, $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); +my $b1name = $ipprc->filename("PPIMAGE.BIN1", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); +my $b2name = $ipprc->filename("PPIMAGE.BIN2", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); +my $statsName = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); + +# Run normalisation +unless ($no_op) { + + # we cannot use ppImage to load a normalized mask : just copy it and build the jpeg images + if ($det_type_real eq 'mask') { + $RECIPE_PPIMAGE = 'PPIMAGE_BIN'; + my $input_real = $ipprc->file_resolve($input_uri, 0); + my $output_real = $ipprc->file_resolve($output, 1); + system ("cp $input_real $output_real"); + } + + my $command = "$ppImage -file $input_uri $outroot"; + $command .= " -norm $value -stats $statsName"; + $command .= " -recipe PPIMAGE $RECIPE_PPIMAGE"; + $command .= " -recipe PPSTATS $RECIPE_PPSTATS"; + $command .= " -F PPIMAGE.OUTPUT $outFile" unless $outFile eq 'PPIMAGE.OUTPUT'; + $command .= ' -isfringe' if $det_type_real eq 'fringe'; + $command .= ' -isdark' if $det_type_real eq 'dark'; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $iter, $class_id, $error_code); + } + &my_die("Can't find expected output file: $output", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($output); + &my_die("Can't find expected output file: $b1name", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($b1name); + &my_die("Can't find expected output file: $b2name", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($b2name); + &my_die("Can't find expected output file: $statsName", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($statsName); + + # Get the statistics on the normalised image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($statsName) or &my_die("Can't open statistics file $statsName: $!\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); + &my_die("Can't open statistics file $statsName: $!\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless defined $statsFile; + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + # parse the statistics MDC file + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents); + unless ($metadata) { + &my_die("Unable to parse metadata config", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR); + } + + # extract the stats from the metadata + unless ($stats->parse($metadata)) { + &my_die("Unable to find all values in statistics output.", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR); + } +} + +# Command to update the database +my $command = "$dettool -addnormalizedimfile"; +$command .= " -det_id $det_id"; +$command .= " -iteration $iter"; +$command .= " -class_id $class_id"; +$command .= " -uri $output"; +$command .= " -path_base $outroot"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addnormalizedimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and defined $class_id and not $no_update) { + my $command = "$dettool -addnormalizedimfile"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -class_id $class_id"; + $command .= " -path_base $outroot"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_norm_calc.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_norm_calc.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_norm_calc.pl (revision 22161) @@ -0,0 +1,253 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run ); +use IPC::Run 0.36 qw( run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse command-line arguments +my ($det_id, $iter, $detType, $outroot, $dbname, $verbose, $no_update, $no_op, $redirect ); +GetOptions( + 'det_id|d=s' => \$det_id, # Detrend id + 'iteration|i=s' => \$iter, # Iteration + 'det_type|t=s' => \$detType, # Detrend type + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do operations + 'redirect-output' => \$redirect, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options --det_id --iteration --det_type --outroot", + -exitval => 3, + ) unless + defined $det_id and + defined $iter and + defined $detType and + defined $outroot; + +my $logfile = $outroot . ".log"; + +$ipprc->redirect_output($logfile) if $redirect; + +use constant STATISTIC => 'bg'; # Background statistic to use from the database + +# Define which detrend types we normalise +use constant NORMALIZE => { + 'bias' => 0, + 'dark' => 0, + 'dark_premask' => 0, + 'shutter' => 0, + 'flat_premask' => 1, + 'domeflat_premask' => 1, + 'skyflat_premask' => 1, + 'flat_raw' => 1, + 'domeflat_raw' => 1, + 'skyflat_raw' => 1, + 'flat' => 1, + 'domeflat' => 1, + 'skyflat' => 1, + 'fringe' => 0, + 'mask' => 0, + 'darkmask' => 0, + 'flatmask' => 0, + }; + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppNormCalc = can_run('ppNormCalc') or (warn "Can't find ppNormCalc" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +&my_die("Unrecognised detrend type: $detType", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless exists NORMALIZE()->{lc($detType)}; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get the list of inputs +my @files; # The input files +{ + my $command = "$dettool -processedimfile"; + $command .= " -det_id $det_id"; # Command to run + $command .= " -included"; # only use the inputs for this detrend run to calculate the norm + $command .= " -dbname $dbname" if defined $dbname; + my @command = split /\s+/, $command; + my ( $stdin, $stdout, $stderr ); # Buffers for running program + print "Running [$command]...\n" if $verbose; + if (not run(\@command, \$stdin, \$stdout, \$stderr)) { + &my_die("Unable to perform dettool -processedimfile on detrend $det_id/$iter: $?", + $det_id, $iter, $PS_EXIT_SYS_ERROR); + } + print $stdout . "\n" if $verbose; + + # Because of the length, need to split into individual metadatas --- it parses SO much quicker! + my @whole = split /\n/, $stdout; + my @single = (); + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ); + &my_die("Unable to parse output from dettool", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless + defined $list; + push @files, @$list; + @single = (); + } + } +} + +my $norms; # MDC with normalisations +if (NORMALIZE()->{lc($detType)} and not $no_op) { + + my %matrix; # Matrix of statistics as a function of exposures and classes + foreach my $file (@files) { + my $exp_id = $file->{'exp_id'}; # Exposure ID + my $class_id = $file->{'class_id'}; # Class ID + my $stat = $file->{STATISTIC()}; # Statistic of interest + + # Create matrix elements + $matrix{$exp_id} = {} if not defined $matrix{$exp_id}; + $matrix{$exp_id}->{$class_id} = $stat; + } + + # Generate the input for ppNormCalc + my $normData; # Normalisation data + foreach my $exp_id (keys %matrix) { + $normData .= "$exp_id\tMETADATA\n"; + foreach my $class_id (keys %{$matrix{$exp_id}}) { + $normData .= "\t" . $class_id . "\tF32\t" . $matrix{$exp_id}->{$class_id} . "\n"; + } + $normData .= "END\n\n"; + } + + # Run ppNormCalc + { + my ( $stdout, $stderr ); # Buffers for running program + my @command = split /\s+/, $ppNormCalc; + print "Running [$ppNormCalc]...\n" if $verbose; + if (not run(\@command, \$normData, \$stdout, \$stderr)) { + &my_die("Unable to perform ppNormCalc: $?", $det_id, $iter, $PS_EXIT_SYS_ERROR); + } + print $stdout . "\n" if $verbose; + + # Parse the output + $norms = $mdcParser->parse($stdout); + &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless $norms; + } + +} else { + # It's something that doesn't need normalisation --- just push in a normalisation of 1 + my %classes; # List of unique classes + foreach my $file (@files) { + my $class_id = $file->{'class_id'}; # Class Id + $classes{$class_id} = 1; + } + + foreach my $class_id (keys %classes) { + my %mdValue; # Metadata value for this class id + $mdValue{name} = $class_id; + $mdValue{value} = 1.0; + push @$norms, \%mdValue; + } +} + +my $commandBase = "$dettool -addnormalizedstat"; +$commandBase .= " -det_id $det_id"; +$commandBase .= " -iteration $iter"; +$commandBase .= " -dbname $dbname" if defined $dbname; + +# Process output normalisations +unless ($no_update) { + foreach my $normItem (@$norms) { + + my $className = $normItem->{name}; # Name of component + my $normalisation = $normItem->{value}; # Normalisation for component + + if ($normalisation == 0.0 or lc($normalisation) eq 'nan') { + warn("Class $className has bad normalisation: $normalisation"); + exit($PS_EXIT_SYS_ERROR); + } + + my $command = $commandBase; + $command .= " -class_id $className"; + $command .= " -norm $normalisation"; + + my @command = split /\s+/, $command; + + my ( $stdin, $stdout, $stderr ); # Buffers for running program + print "Running [$command]...\n" if $verbose; + if (not run \@command, \$stdin, \$stdout, \$stderr) { + warn("Unable to perform dettool -addnormstat for $className: $?"); + exit($PS_EXIT_SYS_ERROR); + } + print $stdout . "\n" if $verbose; + } +} else { + print "skipping command: $commandBase\n"; + foreach my $normItem (@$norms) { + + my $className = $normItem->{name}; # Name of component + my $normalisation = $normItem->{value}; # Normalisation for component + + if ($normalisation == 0.0 or lc($normalisation) eq 'nan') { + warn("Class $className has bad normalisation: $normalisation"); + exit($PS_EXIT_SYS_ERROR); + } + print "$className : $normalisation\n"; + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and not $no_update) { + my $command = "$dettool -addnormalizedstat"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + # XXX EAM : we should add this to the db : $command .= " -path_base $outroot"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_norm_exp.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_norm_exp.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_norm_exp.pl (revision 22161) @@ -0,0 +1,205 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); +use PS::IPP::Metadata::List qw( parse_md_list ); +use File::Temp qw( tempfile ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($det_id, $iter, $det_type, $camera, $outroot, $dbname, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration|i=s' => \$iter, + 'camera|c=s' => \$camera, + 'det_type|t=s' => \$det_type, + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction|=s' => \$reduction, + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --camera --det_type --outroot", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $camera and + defined $det_type and + defined $outroot; + +my $logfile = $outroot . ".log"; + +$ipprc->redirect_output($logfile) if $redirect; + +$ipprc->define_camera($camera); + +# Recipes to use based on reduction class +$reduction = 'DETREND' unless defined $reduction; + +my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1; + +my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_IMAGE_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2; + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + # { name => "bg_mean_stdev", type => "rms", flag => "-bg_mean_stdev" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component files +my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + $command = "$dettool -normalizedimfile"; # Command to run + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to get list of normalized imfiles from dettool: $error_code", $det_id, $iter, $error_code); + } + + # convert stdout to a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # parse the file info in the metadata + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # parse the stats in the metadata + unless ($stats->parse($metadata)) { + &my_die("Unable to find all values in statistics output.\n", $det_id, $iter, $PS_EXIT_PROG_ERROR); + } +} + +my ($list1File, $list1Name) = tempfile( "/tmp/$camera.$det_type.norm.$det_id.$iter.b1.list.XXXX", UNLINK => !$save_temps ); +my ($list2File, $list2Name) = tempfile( "/tmp/$camera.$det_type.norm.$det_id.$iter.b2.list.XXXX", UNLINK => !$save_temps ); +foreach my $file (@$files) { + print $list1File ( $ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n"); + print $list2File ( $ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n"); +} +close $list1File; +close $list2File; + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $jpeg1Name = $ipprc->filename("PPIMAGE.JPEG1", $outroot); # Binned JPEG #1 +my $jpeg2Name = $ipprc->filename("PPIMAGE.JPEG2", $outroot); # Binned JPEG #2 + +unless ($no_op) { + # Make the jpeg for binning 1 + $command = "$ppImage -list $list1Name $outroot -recipe PPIMAGE $recipe1"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die("Unable to find expected output file: $jpeg1Name", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg1Name); + + # Make the jpeg for binning 2 + $command = "$ppImage -list $list2Name $outroot -recipe PPIMAGE $recipe2"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die("Unable to find expected output file: $jpeg2Name", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg2Name); + +} + +# command to update the database +$command = "$dettool -addnormalizedexp"; +$command .= " -det_id $det_id"; +$command .= " -iteration $iter"; +$command .= " -recip $recipe1,$recipe2"; +$command .= " -path_base $outroot "; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -addnormalizedexp: $error_code", $det_id, $iter, $error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and not $no_update) { + my $command = "$dettool -addnormalizedexp"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -path_base $outroot "; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_process_exp.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_process_exp.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_process_exp.pl (revision 22161) @@ -0,0 +1,216 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); +use PS::IPP::Metadata::List qw( parse_md_list ); +use File::Temp qw( tempfile ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $det_id, $exp_id, $det_type, $exp_tag, $camera, $outroot, $dbname, $reduction, $verbose, $no_update, + $no_op, $save_temps, $redirect ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'det_type|t=s' => \$det_type, + 'exp_id|d=s' => \$exp_id, + 'exp_tag|=s' => \$exp_tag, + 'camera|c=s' => \$camera, + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction|=s' => \$reduction, + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'save-temps' => \$save_temps, # Save temporary files? + 'redirect-output' => \$redirect, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --det_type --exp_id --exp_tag --camera --outroot", + -exitval => 3) unless + defined $det_id and + defined $det_type and + defined $exp_id and + defined $exp_tag and + defined $camera and + defined $outroot; + +$ipprc->define_camera($camera); + +if ($redirect) { + my $logDest = $ipprc->filename("LOG.EXP", $outroot, "NONE") + or &my_die("Missing entry in file rules", $det_id, $exp_id, $PS_EXIT_CONFIG_ERROR); + $ipprc->redirect_output($logDest); +} + +# Recipes to use based on reduction class +$reduction = 'DETREND' unless defined $reduction; + +my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $exp_id, $PS_EXIT_PROG_ERROR) unless defined $recipe1; + +my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_IMAGE_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $exp_id, $PS_EXIT_PROG_ERROR) unless defined $recipe2; + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + # { name => "bg_mean_stdev", type => "rms", flag => "-bg_mean_stdev" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component files +my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + $command = "$dettool -processedimfile -det_id $det_id -exp_id $exp_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -processedimfile: $error_code", $det_id, $exp_id, $error_code); + } + + # convert stdout to a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $exp_id, $PS_EXIT_PROG_ERROR); + + # parse the file info in the metadata + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $exp_id, $PS_EXIT_PROG_ERROR); + + # parse the stats in the metadata + unless ($stats->parse($metadata)) { + &my_die("Unable to find all values in statistics output.\n", $det_id, $exp_id, $PS_EXIT_PROG_ERROR); + } +} + +# File lists for the two binnings +my ($list1File, $list1Name) = tempfile( "/tmp/$exp_tag.detproc.$det_id.b1.list.XXXX", UNLINK => !$save_temps ); +my ($list2File, $list2Name) = tempfile( "/tmp/$exp_tag.detproc.$det_id.b2.list.XXXX", UNLINK => !$save_temps ); +foreach my $file (@$files) { + print $list1File ($ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n"); + print $list2File ($ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n"); +} +close $list1File; +close $list2File; + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $jpeg1 = $ipprc->filename("PPIMAGE.JPEG1", $outroot); # Binned JPEG #1 +my $jpeg2 = $ipprc->filename("PPIMAGE.JPEG2", $outroot); # Binned JPEG #2 + +unless ($no_op) { + # Make the jpeg for binning 1 + $command = "$ppImage -list $list1Name $outroot -recipe PPIMAGE $recipe1"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg1", $det_id, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg1); + + # Make the jpeg for binning 2 + $command = "$ppImage -list $list2Name $outroot -recipe PPIMAGE $recipe2"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg2", $det_id, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg2); +} + +# Command to update the database +$command = "$dettool -addprocessedexp"; +$command .= " -det_id $det_id"; +$command .= " -exp_id $exp_id"; +$command .= " -recip $recipe1,$recipe2 -path_base $outroot"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addprocessedexp: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $exp_id = shift; # Exposure tag + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $exp_id and not $no_update) { + my $command = "$dettool -addprocessedexp"; + $command .= " -det_id $det_id"; + $command .= " -exp_id $exp_id"; + $command .= " -code $exit_code"; + $command .= " -path_base $outroot"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_process_imfile.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_process_imfile.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_process_imfile.pl (revision 22161) @@ -0,0 +1,192 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $det_id, $exp_id, $class_id, $det_type, $exp_tag, $input_uri, $camera, $outroot, $dbname, $reduction, + $threads, $verbose, $no_update, $no_op, $redirect ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'exp_id|e=s' => \$exp_id, + 'class_id|i=s' => \$class_id, + 'det_type|t=s' => \$det_type, + 'exp_tag|=s' => \$exp_tag, + 'input_uri|u=s' => \$input_uri, + 'camera|c=s' => \$camera, + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction=s' => \$reduction, # Reduction class + 'threads=s' => \$threads, # Number of threads to use for ppImage + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'redirect-output' => \$redirect, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --exp_id --class_id --det_type --exp_tag --input_uri --camera --outroot", + -exitval => 3) unless + defined $det_id and + defined $exp_id and + defined $class_id and + defined $det_type and + defined $exp_tag and + defined $input_uri and + defined $camera and + defined $outroot; + +# XXX this exits with status = 0 on failure +$ipprc->define_camera($camera); + +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_CONFIG_ERROR); + +$ipprc->redirect_output($logDest) if $redirect; + +# Recipes to use as a function of detrend type +$reduction = "DETREND" unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, uc($det_type) . '_PROCESS'); # Recipe name to use + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +&my_die("Couldn't find input file: $input_uri\n", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input_uri); + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $outputImage = $ipprc->filename("PPIMAGE.OUTPUT", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +my $outputBin1 = $ipprc->filename("PPIMAGE.BIN1", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +my $outputBin2 = $ipprc->filename("PPIMAGE.BIN2", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +my $outputStats = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $exp_id, $class_id, $PS_EXIT_CONFIG_ERROR); +# Run ppImage +unless ($no_op) { + my $command = "$ppImage -file $input_uri $outroot"; + $command .= " -recipe PPIMAGE $recipe"; + $command .= " -recipe PPSTATS CHIPSTATS"; + $command .= " -stats $outputStats"; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -threads $threads" if defined $threads; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $class_id, $error_code); + } + + &my_die("Couldn't find expected output file: $outputImage", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputImage); + &my_die("Couldn't find expected output file: $outputStats", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $outputBin1", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputBin1); + &my_die("Couldn't find expected output file: $outputBin2", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputBin2); + + # Get the statistics on the processed image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve("$outputStats") or die "Can't open statistics file $outputStats: $!\n"; + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + # parse the statistics MDC file + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents) + or &my_die("Unable to parse metadata config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + + # extract the stats from the metadata + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +} + +# command to update database +my $command = "$dettool -addprocessedimfile"; +$command .= " -det_id $det_id"; +$command .= " -exp_id $exp_id"; +$command .= " -class_id $class_id"; +$command .= " -recip $reduction"; +$command .= " -uri $outputImage -path_base $outroot"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addprocessedimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $exp_id = shift; # Exposure tag + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $exp_id and defined $class_id and not $no_update) { + my $command = "$dettool -addprocessedimfile"; + $command .= " -det_id $det_id"; + $command .= " -exp_id $exp_id"; + $command .= " -class_id $class_id"; + $command .= " -path_base $outroot"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_reject_exp.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_reject_exp.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_reject_exp.pl (revision 22161) @@ -0,0 +1,371 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); +use PS::IPP::Metadata::List qw( parse_md_list ); +use Statistics::Descriptive; + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my $ITER_LIMIT = 20; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($det_id, $iter, $det_type, $camera, $outroot, $filter, $dbname, $verbose, $no_update, $no_op, $redirect); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration=s' => \$iter, + 'det_type|t=s' => \$det_type, + 'camera=s' => \$camera, + 'outroot|w=s' => \$outroot, # output file base name + 'filter=s' => \$filter, + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'redirect-output' => \$redirect, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --det_type --camera --outroot", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $det_type and + defined $camera and + defined $outroot; + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $logName = "$outroot.log"; # Name for log + +$ipprc->redirect_output($logName) if $redirect; + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg_mean_stdev", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# these stats are used it the rejections but not passed to the database +# there is some duplication with the above, but the calculation time is minimal +my $REJSTATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "clipmean", flag => "ensMeanMean", dtype => "float" }, + { name => "bg", type => "clipstdev", flag => "ensMeanStdev", dtype => "float" }, + { name => "bg_mean_stdev", type => "clipmean", flag => "ensMeanStdevMean", dtype => "float" }, + { name => "bg_mean_stdev", type => "clipstdev", flag => "ensMeanStdevStdev", dtype => "float" }, + { name => "bg_stdev", type => "clipmean", flag => "ensStdevMean", dtype => "float" }, + { name => "bg_stdev", type => "clipstdev", flag => "ensStdevStdev", dtype => "float" }, + { name => "bg_skewness", type => "clipmean", flag => "ensSkewness", dtype => "float" }, + { name => "bg_kurtosis", type => "clipmean", flag => "ensKurtosis", dtype => "float" }, + + ]; +my $rejstats = PS::IPP::Metadata::Stats->new($REJSTATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component files +my ($exposures, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + # dettool command to select exp data for this det_run + $command = "$dettool -residexp"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -residexp: $error_code", $det_id, $iter, $error_code); + } + + # Parse the stdout buffer into a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # parse the file info in the metadata + $exposures = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # Parse the statistics on the residual image + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # Parse the statistics for rejections + $rejstats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $PS_EXIT_PROG_ERROR); +} + +# each image has a mean mean (average of means over all chips) +# a standard deviation mean (average of standard deviations over all chips) +# and a + + + +# we use the statistics of the ensemble to accept/reject exposurs +my $ensMeanMean = $rejstats->value_for_flag ("ensMeanMean"); # average of all exposure means (in turn averaged over all chips) +my $ensMeanStdev = $rejstats->value_for_flag ("ensMeanStdev"); # standard deviation of all exposure means +my $ensMeanStdevMean = $rejstats->value_for_flag ("ensMeanStdevMean"); # average over all exposures of the stdev of the means for each chip +my $ensMeanStdevStdev = $rejstats->value_for_flag ("ensMeanStdevStdev"); # standard deviation over all exposures of the stdev of the means for each chip +my $ensStdevMean = $rejstats->value_for_flag ("ensStdevMean"); # average over all exposures of the sum of the squares of the stdevs for each chip +my $ensStdevStdev = $rejstats->value_for_flag ("ensStdevStdev"); # standard deviation over all exposures of the sum of the squares of the stdevs for each chip + +$ipprc->define_camera($camera); +# Rejection thresholds +my $reject_mean = rejection_limit( 'ENSEMBLE.MEAN', $det_type, $filter ); +my $reject_stdev = rejection_limit( 'ENSEMBLE.STDEV', $det_type, $filter ); +my $reject_meanstdev = rejection_limit( 'ENSEMBLE.MEANSTDEV', $det_type, $filter ); + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +unless ($no_op) { + print "Ensemble mean $ensMeanMean +/- $ensMeanStdev\n"; + print "Ensemble stdev $ensStdevMean +/- $ensStdevStdev\n"; + print "Ensemble mean rms (over imfiles) $ensMeanStdevMean +/- $ensMeanStdevStdev\n\n"; +} + +# Go through again to do rejection, and update the database for each exposure +my $numChanges = 0; # Number of exposures with changed status +my $numReject = 0; # Number of exposures rejected +my $firstElement = 1; + +foreach my $exposure (@$exposures) { + my $mean = $exposure->{bg}; # Mean for this exposure + my $stdev = $exposure->{bg_stdev}; # Stdev for this exposure + my $meanStdev = $exposure->{bg_mean_stdev}; # Stdev of Means for this exposure + my $expID = $exposure->{exp_id}; + my $accept = $exposure->{accept}; + my $include = $exposure->{include}; + + &my_die("Unable to find exposure id.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $expID; + &my_die("Unable to find accept.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $accept; + &my_die("Unable to find include.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $include; + + my $reject = 0; # Reject this exposure? + + $command = "$dettool -updateresidexp"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -exp_id $expID"; + $command .= " -dbname $dbname" if defined $dbname; + + if (not $accept) { + # Rejected this at an earlier stage + unless ($no_op) { + print "Rejecting $expID based on earlier determination.\n"; + } + $reject = 1; + goto UPDATE; + } + + # Cop-out if we're not doing any operations + if ($no_op) { + # Make sure something gets rejected (just once!), just so that + # we can trace the full range of the workflow + if ($firstElement and $iter == 0) { + $reject = 1; + } + goto UPDATE; + } + + if ($reject_mean > 0 and $ensMeanStdev > 0) { + my $delta = abs($mean - $ensMeanMean); + if ($delta > ($reject_mean * $ensMeanStdev)) { + print "Rejecting $expID based on ensemble mean value: "; + $reject = 1; + #goto UPDATE; + } else { + print "$expID OK against ensemble mean: "; + } + print "$mean --> $delta vs " . $reject_mean * $ensMeanStdev . "\n"; + } else { + print "No rejection of $expID for ensemble mean\n"; + } + + if ($reject_stdev > 0 and $ensStdevStdev > 0) { + my $delta = abs($stdev - $ensStdevMean); + if ($delta > ($reject_stdev * $ensStdevStdev)) { + print "Rejecting $expID based on ensemble stdev: "; + $reject = 1; + #goto UPDATE; + } else { + print "$expID OK against ensemble stdev: "; + } + print "$stdev --> $delta sigma vs " . $reject_stdev * $ensStdevStdev . "\n"; + } else { + print "No rejection of $expID for ensemble stdev\n"; + } + + if ($reject_meanstdev > 0 and $ensMeanStdevStdev > 0) { + my $delta = abs($meanStdev - $ensMeanStdevMean); + if ($delta > ($reject_meanstdev * $ensMeanStdevStdev)) { + print "Rejecting $expID based on ensemble mean stdev: "; + $reject = 1; + #goto UPDATE; + } else { + print "$expID OK against ensemble mean stdev: "; + } + print "$meanStdev --> $delta sigma vs " . $reject_meanstdev * $ensMeanStdevStdev. "\n"; + } else { + print "No rejection of $expID for ensemble mean stdev\n"; + } + + UPDATE: + if ($reject) { + $command .= ' -reject'; + $numReject++; + } + + # Check for status changes + if ((not $include and not $reject) or ($include and $reject)) { + unless ($no_op) { + print "Status of $expID has changed.\n"; + } + $numChanges++; + } + + unless ($no_update) { + # Update + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -updateresidexp: $error_code", $det_id, $iter, $error_code); + } + } +} + +# Decide if the current is sufficient to use as a master, and if we can stop iterating +my $master = 1; # This is good enough for a master +my $stop = 1; # Stop iterating + +if ($numChanges > 0) { + $master = 0; + $stop = 0; +} + +# Rejecting everything --- stop before something bad happens! +if ($numReject == scalar @$exposures) { + $master = 0; + $stop = 1; + carp "All inputs rejected!\n"; +} + +unless ($no_op) { + print "Master: $master\n"; + print "Stop: $stop\n"; +} + +# Allow iteration to be turned off +my $allow_iter = metadataLookupBool($ipprc->{rejection}, "ITERATION"); # Allow iteration? +my $force_master = metadataLookupBool($ipprc->{rejection}, "MASTER"); # Force the stack to be accepted +$stop = 1 unless $allow_iter; +$master = 1 if $force_master and $stop; + +# attempt to prevent endless, pathological iterations +if ($iter >= $ITER_LIMIT) { + warn("iteration limit reached -- bailing out"); + exit($PS_EXIT_PROG_ERROR); +} + +## add the summary statistics, and request a new iteration if needed +$command = "$dettool -adddetrunsummary"; +$command .= " -det_id $det_id"; +$command .= " -iteration $iter"; +$command .= " -accept" if $master; +$command .= " -dbname $dbname" if defined $dbname; +$command .= " -again" unless $stop; +$command .= $stats->cmdflags(); + +# Put results into the database +unless ($no_update) { + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -adddetrunsummary: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and not $no_update) { + my $command = "$dettool -adddetrunsummary"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + # XXX EAM : we should add this to the db : $command .= " -path_base $outroot"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Retrieve the requested rejection limit, dying unless extant +sub rejection_limit +{ + my $name = shift; # Rejection limit to + my $type = shift; # Type of exposure + my $filter = shift; # Filter + + my $value = $ipprc->rejection( $name, $det_type, $filter ); + if (not defined $value) { + $filter = "(no filter)" unless defined $filter; + die "Unable to determine $name rejection limit for $det_type with $filter.\n"; + } + + return $value; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_resid_exp.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_resid_exp.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_resid_exp.pl (revision 22161) @@ -0,0 +1,601 @@ +#!/usr/bin/env perl + +# this program has two jobs: +# 1: for the given exp_tag, generate the binned & mosaiced JPEG images +# 2: examine the collection of per-imfile statistics and reject. At the moment, +# this program (and the database) only allows rejection at the exposure level, +# not at the imfile level. + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); # tools to run UNIX programs with control over I/O +use PS::IPP::Metadata::Config; # tools to parse the psMetadataConfig files +use PS::IPP::Metadata::Stats; + +use PS::IPP::Metadata::List qw( parse_md_list ); # tools to parse a metadata into a hash list +use Statistics::Descriptive; # tools for calculating basic statistical quantities +use File::Temp qw( tempfile ); # tools to construct temp files +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); # option parsing +use Pod::Usage qw( pod2usage ); + +# parse the command-line options +my ( $det_id, $iter, $exp_id, $exp_tag, $det_type, $camera, $filter, $reject, $outroot, $dbname, $reduction, + $verbose, $no_update, $no_op, $save_temps, $redirect ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration=s' => \$iter, + 'exp_id|e=s' => \$exp_id, + 'exp_tag|=s' => \$exp_tag, + 'det_type|t=s' => \$det_type, + 'camera=s' => \$camera, + 'outroot|w=s' => \$outroot, # output file base name + 'filter=s' => \$filter, + 'reject' => \$reject, + 'dbname|d=s' => \$dbname, # Database name + 'reduction|=s' => \$reduction, + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'save-temps' => \$save_temps, # Save temporary files? + 'redirect-output' => \$redirect, # redirect output to LOG.EXP + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --exp_id --exp_tag --det_type --camera --outroot", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $exp_id and + defined $exp_tag and + defined $det_type and + defined $camera and + defined $outroot; + +# load IPP config information for the specified camera +$ipprc->define_camera($camera); +if ($redirect) { + my $logDest = $ipprc->filename("LOG.EXP", $outroot) + or &my_die("Missing entry from camera config", $det_id, $iter, $exp_id, $PS_EXIT_CONFIG_ERROR); + $ipprc->redirect_output($logDest); +} + +# Recipes to use based on reduction class +$reduction = 'DETREND' unless defined $reduction; + +my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_RESID_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1; + +my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_RESID_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2; + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg_mean_stdev", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "bg_skewness", type => "mean", flag => "-bg_skewness", dtype => "float" }, + { name => "bg_kurtosis", type => "mean", flag => "-bg_kurtosis", dtype => "float" }, + { name => "bin_stdev", type => "rms", flag => "-bin_stdev", dtype => "float" }, + { name => "fringe_0", type => "mean", flag => "-fringe_0", dtype => "float" }, + { name => "fringe_1", type => "rms", flag => "-fringe_1", dtype => "float" }, + { name => "fringe_2", type => "stdev", flag => "-fringe_2", dtype => "float" }, + { name => "user_1", type => "mean", flag => "-user_1", dtype => "float" }, # fringe residual + { name => "user_2", type => "rms", flag => "-user_2", dtype => "float" }, # fringe residual + { name => "user_1", type => "stdev", flag => "-user_3", dtype => "float" }, # fringe residual + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of imfile files +my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + # dettool command to select imfile data for this exp_id + $command = "$dettool -residimfile"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -exp_id $exp_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -residimfile: $error_code\n"); + exit($error_code); + } + # XXX report an error message if stdout_buf is empty + + # Parse the stdout buffer into a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR); + + # parse the file info in the metadata + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR); + + # Parse the statistics on the residual image + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR); +} + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $jpeg1Name = $ipprc->filename("PPIMAGE.JPEG1", $outroot); # Binned JPEG #1 +my $jpeg2Name = $ipprc->filename("PPIMAGE.JPEG2", $outroot); # Binned JPEG #2 + +# XXX in debug mode, unlink = 0 +my ($list1File, $list1Name) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.b1.list.XXXX", UNLINK => !$save_temps ); +my ($list2File, $list2Name) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.b2.list.XXXX", UNLINK => !$save_temps ); +foreach my $file (@$files) { + print $list1File ($ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n"); + print $list2File ($ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n"); +} +close $list1File; +close $list2File; + + +# build the JPEG images +unless ($no_op) { + # Make the jpeg for binning 1 + $command = "$ppImage -list $list1Name $outroot -recipe PPIMAGE $recipe1"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run ppImage: $error_code", $det_id, $iter, $exp_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg1Name", $det_id, $iter, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg1Name); + + # Make the jpeg for binning 2 + $command = "$ppImage -list $list2Name $outroot -recipe PPIMAGE $recipe2"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run ppImage: $error_code", $det_id, $iter, $exp_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg2Name", $det_id, $iter, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg2Name); +} + +#### measure stats and reject if needed #### + +# Rejection thresholds & related data +my $expected = rejection_limit( 'EXPECTED', $det_type, $filter ); +my $reject_imfile_mean = rejection_limit( 'IMFILE.MEAN', $det_type, $filter ); +my $reject_imfile_flux = rejection_limit( 'IMFILE.FLUX', $det_type, $filter ); +my $reject_imfile_stdev = rejection_limit( 'IMFILE.STDEV', $det_type, $filter ); +my $reject_imfile_skewness = rejection_limit( 'IMFILE.SKEWNESS', $det_type, $filter ); +my $reject_imfile_kurtosis = rejection_limit( 'IMFILE.KURTOSIS', $det_type, $filter ); +my $reject_imfile_meanstdev = rejection_limit( 'IMFILE.MEANSTDEV', $det_type, $filter ); +my $reject_imfile_snr = rejection_limit( 'IMFILE.SNR', $det_type, $filter ); +my $reject_imfile_bin_stdev = rejection_limit( 'IMFILE.BIN.STDEV', $det_type, $filter ); +my $reject_imfile_bin_snr = rejection_limit( 'IMFILE.BIN.SNR', $det_type, $filter ); +my $reject_exp_mean = rejection_limit( 'EXP.MEAN', $det_type, $filter ); +my $reject_exp_flux = rejection_limit( 'EXP.FLUX', $det_type, $filter ); +my $reject_exp_stdev = rejection_limit( 'EXP.STDEV', $det_type, $filter ); +my $reject_exp_meanstdev = rejection_limit( 'EXP.MEANSTDEV', $det_type, $filter ); +my $reject_exp_snr = rejection_limit( 'EXP.SNR', $det_type, $filter ); +my $reject_exp_bin_stdev = rejection_limit( 'EXP.BIN.STDEV', $det_type, $filter ); +my $reject_exp_bin_snr = rejection_limit( 'EXP.BIN.SNR', $det_type, $filter ); + +# storage array +my @fluxes; + +foreach my $file (@$files) { + my $name = $file->{class_id}; + my $mean = $file->{bg}; # Mean for this imfile + my $stdev = $file->{bg_stdev}; # Stdev for this imfile + my $skewness = $file->{bg_skewness}; # Skewness for this imfile + my $kurtosis = $file->{bg_kurtosis}; # Kurtosis for this imfile + my $meanStdev = $file->{bg_mean_stdev}; # Stdev of Means for this imfile + my $binStdev = $file->{bin_stdev}; # Binned Stdev for this imfile + + # calculate and save the fluxes + my $flux; + if ($file->{exp_time} == 0.0) { + $flux = $mean; + } else { + $flux = $mean / $file->{exp_time}; + } + push @fluxes, $flux; + + $mean -= $expected; + + last if $no_op; + + # reject exposure if, for any imfiles, the mean residual counts + # deviate from the expected value by more than N sigma, (sigma = + # total pixel variance). this test is sensible for images which + # should have a predictable nominal residual mean count value (eg, + # 0.0 for a bias). in general, use with ADDITIVE components + if ($reject_imfile_mean > 0) { + if (abs($mean) > $reject_imfile_mean * $stdev) { + print "Rejecting exposure based on bad imfile mean for $name: "; + $reject = 1; + } else { + print "Imfile mean for $name meets requirements: "; + } + print "$mean vs $reject_imfile_mean * $stdev\n"; + } else { + print "No rejection on imfile mean for $name\n"; + } + + # reject exposure if, for any imfiles, the mean residual flux + # deviates from the expected value by more than N sigma, (sigma = + # total pixel variance). this test is sensible for images which + # should have a predictable nominal residual flux value (eg, 0.0 + # for a bias). in general, use with ADDITIVE components + if ($reject_imfile_flux > 0) { + if (abs($flux) > $reject_imfile_flux) { + print "Rejecting exposure based on bad imfile flux for $name: "; + $reject = 1; + } else { + print "Imfile flux for $name meets requirements: "; + } + print "$flux vs $reject_imfile_flux\n"; + } else { + print "No rejection on imfile flux for $name\n"; + } + + # reject exposure if, for any imfiles, the total pixel variance is + # larger than the limit + if ($reject_imfile_stdev > 0) { + if ($stdev > $reject_imfile_stdev) { + print "Rejecting exposure based on bad imfile stdev for $name: "; + $reject = 1; + } else { + print "Imfile stdev for $name meets requirements: "; + } + print "$stdev vs $reject_imfile_stdev\n"; + + } else { + print "No rejection on imfile stdev for $name\n"; + } + + # reject exposure if, for any imfiles, the skewness is + # larger than the limit + if ($reject_imfile_skewness > 0) { + if ($stdev > $reject_imfile_skewness) { + print "Rejecting exposure based on bad imfile skewness for $name: "; + $reject = 1; + } else { + print "Imfile skewness for $name meets requirements: "; + } + print "$skewness vs $reject_imfile_skewness\n"; + + } else { + print "No rejection on imfile skewness for $name\n"; + } + + # reject exposure if, for any imfiles, the kurtosis is + # larger than the limit + if ($reject_imfile_kurtosis > 0) { + if ($stdev > $reject_imfile_kurtosis) { + print "Rejecting exposure based on bad imfile kurtosis for $name: "; + $reject = 1; + } else { + print "Imfile kurtosis for $name meets requirements: "; + } + print "$kurtosis vs $reject_imfile_kurtosis\n"; + + } else { + print "No rejection on imfile kurtosis for $name\n"; + } + + # reject exposure if, for any imfiles, the variance of the imfile + # component means is larger than the limit + if ($reject_imfile_meanstdev > 0) { + if ($meanStdev > $reject_imfile_meanstdev) { + print "Rejecting exposure based on bad imfile mean stdev for $name: "; + $reject = 1; + } else { + print "Imfile mean stdev for $name meets requirements: "; + } + print "$meanStdev vs $reject_imfile_meanstdev\n"; + } else { + print "No rejection on imfile mean stdev for $name\n"; + } + + # reject exposure if, for any imfiles, the signal-to-noise (ie, + # the mean counts / total pixel variance) of the imfile component + # means are less than the limit. this test is sensible for images + # which have finite residual flux such as a flat-field image. + if ($reject_imfile_snr > 0) { + if ($mean < $stdev * $reject_imfile_snr) { + print "Rejecting exposure based on bad imfile S/N for $name: "; + $reject = 1; + } else { + print "Imfile S/N for $name meets requirements: "; + } + print "mean: $mean vs stdev*SNlimit: " . $stdev * $reject_imfile_snr . "\n"; + } else { + print "No rejection on imfile S/N for $name\n"; + } + + # reject exposure if, for any imfiles, the signal-to-noise of the + # imfile component means, based on the stdev of the binned image + # is less than the limit. this test is sensible for images which + # have finite residual flux such as a flat-field image. + if ($reject_imfile_bin_stdev > 0) { + if ($binStdev > $reject_imfile_bin_stdev) { + print "Rejecting exposure based on bad imfile binned stdev for $name: "; + $reject = 1; + } else { + print "Imfile binned stdev for $name meets requirements: "; + } + print "$binStdev vs $reject_imfile_bin_stdev\n"; + } else { + print "No rejection on imfile binned stdev for $name\n"; + } + if ($reject_imfile_bin_snr > 0) { + if ($mean < $binStdev * $reject_imfile_bin_snr) { + print "Rejecting exposure based on bad imfile binned S/N for $name: "; + $reject = 1; + } else { + print "Imfile binned S/N for $name meets requirements: "; + } + print "mean: $mean vs binStdev*SNlimit: " . $binStdev * $reject_imfile_bin_snr . "\n"; + } else { + print "No rejection on imfile binned S/N for $name\n"; + } +} + +# basic ensemble stats +my $mean = $stats->value_for_flag ("-bg"); +my $meanStdev = $stats->value_for_flag ("-bg_mean_stdev"); +my $stdev = $stats->value_for_flag ("-bg_stdev"); +my $binStdev = $stats->value_for_flag ("-bin_stdev"); +my $fringe_mean = $stats->value_for_flag ("-fringe_0"); +my $fringe_err = $stats->value_for_flag ("-fringe_1"); +my $fringe_mean_stdev = $stats->value_for_flag ("-fringe_2"); +my $dfringe_mean = $stats->value_for_flag ("-fringe_resid_0"); +my $dfringe_err = $stats->value_for_flag ("-fringe_resid_1"); +my $dfringe_mean_stdev = $stats->value_for_flag ("-fringe_resid_2"); + +# other stats (flux depends on bg and exp_time) +my $fluxStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator for means +$fluxStats->add_data(@fluxes); +my $flux = $fluxStats->mean(); # Mean of the imfile means + +# other stats +my $exp_snr = 0.0; +if ($stdev > 0) { $exp_snr = $mean / $stdev; } + +## Reject based on the exposure ensemble stats +# reject if the exposure ensemble mean is deviant +unless ($no_op) { + print "Exposure mean $mean, stdev $stdev, mean stdev $meanStdev, exp s/n: $exp_snr\n"; + + # reject exposure if the ensemble mean residual counts deviate + # from the expected value by more than N sigma, (sigma = total + # pixel variance). this test is sensible for images which should + # have a predictable nominal residual mean count value (eg, 0.0 + # for a bias). in general, use with ADDITIVE components + if ($reject_exp_mean > 0) { + if (abs($mean) > $reject_exp_mean * $stdev) { + print "Rejecting exposure based on bad mean counts: "; + $reject = 1; + } else { + print "Exposure mean counts meets requirements: "; + } + print "mean: $mean, stdev: $stdev (limit is: " . $reject_exp_mean * $stdev . ") \n"; + } else { + print "No rejection for exp mean\n"; + } + + # reject exposure if, for any imfiles, the mean residual flux + # deviates from the expected value by more than N sigma, (sigma = + # total pixel variance). this test is sensible for images which + # should have a predictable nominal residual flux value (eg, 0.0 + # for a bias). in general, use with ADDITIVE components + if ($reject_exp_flux > 0) { + if (abs($flux) > $reject_exp_flux * $stdev) { + print "Rejecting exposure based on bad mean flux: "; + $reject = 1; + } else { + print "Exposure mean flux meets requirements: "; + } + print "flux: $flux, stdev: $stdev (limit is: " . $reject_exp_flux * $stdev . ")\n"; + } else { + print "No rejection for exp mean\n"; + } + + # reject exposure if the total pixel variance is larger than the + # limit + if ($reject_exp_stdev > 0) { + if ($stdev > $reject_exp_stdev) { + print "Rejecting exposure based on bad stdev: "; + $reject = 1; + } else { + print "Exposure stdev meets requirements: "; + } + print "$stdev vs $reject_exp_stdev\n"; + } else { + print "No rejection for exp stdev\n"; + } + + # reject exposure if the variance of the imfile means is larger + # than the limit + if ($reject_exp_meanstdev > 0) { + if ($meanStdev > $reject_exp_meanstdev) { + print "Rejecting exposure based on bad mean stdev: "; + $reject = 1; + } else { + print "Exposure mean stdev meets requirements: "; + } + print "$meanStdev vs $reject_exp_meanstdev\n"; + } else { + print "No rejection for exp mean stdev\n"; + } + + # reject if the signal-to-noise is insufficient + if ($reject_exp_snr > 0) { + if (abs($mean) < abs($stdev * $reject_exp_snr)) { + print "Rejecting exposure based on poor S/N: \n"; + $reject = 1; + } else { + print "Exposure S/N meets requirements: \n"; + } + print "signal: $mean vs noise: $stdev (s/n limit is: $reject_exp_snr)\n"; + } else { + print "No rejection for exp S/N\n"; + } + # reject if the exposure ensemble stdev is deviant + if ($reject_exp_bin_stdev > 0) { + if ($binStdev > $reject_exp_bin_stdev) { + print "Rejecting exposure based on bad binned stdev: "; + $reject = 1; + } else { + print "Exposure binned stdev meets requirements: "; + } + print "$stdev vs $reject_exp_bin_stdev\n"; + } else { + print "No rejection for exp binned stdev\n"; + } + # reject if the signal-to-noise is insufficient + if ($reject_exp_bin_snr > 0) { + if (abs($mean) < abs($binStdev * $reject_exp_bin_snr)) { + print "Rejecting exposure based on poor binned S/N: \n"; + $reject = 1; + } else { + print "Exposure binned S/N meets requirements: \n"; + } + print "signal: $mean vs noise: $binStdev (s/n limit is: $reject_exp_bin_snr)\n"; + } else { + print "No rejection for exp binned S/N\n"; + } +} + +$command = "$dettool -addresidexp -det_id $det_id -iteration $iter -exp_id $exp_id"; +$command .= " -recip $recipe1,$recipe2 -path_base $outroot "; +$command .= ' -reject' if $reject; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +unless ($no_update) { + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addresidexp: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exp_id = shift; # Exposure tag + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and defined $exp_id and not $no_update) { + my $command = "$dettool -addresidexp"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -exp_id $exp_id"; + $command .= " -path_base $outroot"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Retrieve the requested rejection limit, dying if not extant +sub rejection_limit +{ + my $name = shift; # Rejection limit to + my $type = shift; # Type of exposure + my $filter = shift; # Filter + + my $value = $ipprc->rejection( $name, $det_type, $filter ); + if (not defined $value) { + $filter = "(no filter)" if not defined $filter; + die "Unable to determine $name rejection limit for $det_type with $filter.\n"; + } + + return $value; +} + +__END__ + +#### + +## this function is not well though out. it loads a collection of +## values (background mean and stdev) for a collection of imfile +## subcomponents. it then measures the mean background, the +## background stdev (rms of ensemble stdevs), and the background mean +## stdev (rms of the means). it then tries to reject based on the +## following criteria: +## (background / stdev > limit) for a single component +## ---> this will ACCEPT for a poor images (large stdev) +## (stdev > limit) +## (mean background / stdev > limit) same problem +## (mean stdev > limit) + +### I would suggest the following: +## 1) calculate the ensemble mean and stdev +## 2) for each component, is the (back - mean) / mean stdev > limit? +## (ie, is it an outlier?) +## 3) is the component stdev > limit? (absolute test) +## 4) calculate the background stdev for the ensemble (excluding rejects) +## 5) is the mean stdev > limit (poor images) +## 6) calculate the ensemble stdev (rms) of remaining (excluding rejects) +## 7) is the ensemble stdev > limit + +## for flats, the value used for stdev should be the fractional stdev +## (stdev/mean) and no rejection should be performed on the basis of +## the mean value, but could still be done on the mean stdev + +## the same logical cuts above can be applied to components in an +## imfile, imfiles in an exposure, and exposures in a stack + +### this function needs to avoid div by zero: compare Signal vs Noise*limit Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_resid_imfile.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_resid_imfile.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_resid_imfile.pl (revision 22161) @@ -0,0 +1,280 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $det_id, $iter, $ref_det_id, $ref_iter, $exp_id, $exp_tag, $class_id, $det_type, $detrend, $input_uri, $camera, $mode, $outroot, + $dbname, $reduction, $threads, $verbose, $no_update, $no_op, $redirect ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration=s' => \$iter, + 'ref_det_id=s' => \$ref_det_id, + 'ref_iter=s' => \$ref_iter, + 'exp_id|e=s' => \$exp_id, + 'exp_tag|=s' => \$exp_tag, + 'class_id|i=s' => \$class_id, + 'det_type|t=s' => \$det_type, + 'detrend=s' => \$detrend, + 'input_uri|u=s' => \$input_uri, + 'camera|c=s' => \$camera, + 'mode|m=s' => \$mode, + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction=s' => \$reduction, # Reduction class + 'threads=s' => \$threads, # Number of threads to use for ppImage + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'redirect-output' => \$redirect, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --ref_det_id --ref_iter --exp_id --exp_tag --class_id --det_type --camera --input_uri --mode --detrend --outroot (not for 'verify' mode)", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $ref_det_id and + defined $ref_iter and + defined $exp_id and + defined $exp_tag and + defined $class_id and + defined $det_type and + defined $input_uri and + defined $camera and + defined $mode and + defined $outroot and + defined $detrend; + +$ipprc->define_camera($camera); +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id); +if ($redirect) { + $ipprc->redirect_output($logDest); +} + +# Recipes to use as a function of detrend type and mode +# XXX probably can drop the distinct 'verify' recipes +$reduction = 'DETREND' unless defined $reduction; +my $recipe; # Name of recipe to use +if ($mode eq 'master') { + $recipe = uc($det_type) . '_RESID'; +} elsif ($mode eq 'verify') { + $recipe = uc($det_type) . '_VERIFY'; +} else { + &my_die("Unrecognised mode: $mode", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +} + +print "raw recipe: $recipe\n"; +$recipe = $ipprc->reduction($reduction, $recipe); +print "real recipe: $recipe\n"; + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "SAMPLE_SKEWNESS", type => "mean", flag => "-bg_skewness", dtype => "float" }, + { name => "SAMPLE_KURTOSIS", type => "mean", flag => "-bg_kurtosis", dtype => "float" }, + { name => "FRINGE_0", type => "mean", flag => "-fringe_0", dtype => "float" }, + { name => "FRINGE_ERR_0", type => "rms", flag => "-fringe_1", dtype => "float" }, + { name => "FRINGE_0", type => "stdev", flag => "-fringe_2", dtype => "float" }, + { name => "FRINGE_RESID_0", type => "mean", flag => "-fringe_resid_0", dtype => "float" }, + { name => "FRINGE_RESID_ERR_0", type => "rms", flag => "-fringe_resid_1", dtype => "float" }, + { name => "FRINGE_RESID_0", type => "stdev", flag => "-fringe_resid_2", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +my $BINNED_STATS = + [ + { name => "ROBUST_STDEV", type => "rms", flag => "-bin_stdev" }, + ]; +my $binnedStats = PS::IPP::Metadata::Stats->new($BINNED_STATS); # Stats parser + +# Flags to specify the particular detrend to use +use constant DETRENDS => { + 'bias' => '-bias', # Specify the bias frame + 'dark' => '-dark', # Specify the dark frame + 'dark_premask' => '-dark', # Specify the dark frame + 'shutter' => '-shutter', # Specify the shutter frame + 'flat_premask' => '-flat', # Specify the flat frame + 'domeflat_premask' => '-flat', # Specify the flat frame + 'skyflat_premask' => '-flat', # Specify the flat frame + 'flat_raw' => '-flat', # Specify the flat frame + 'domeflat_raw' => '-flat', # Specify the flat frame + 'skyflat_raw' => '-flat', # Specify the flat frame + 'flat' => '-flat', # Specify the flat frame + 'domeflat' => '-flat', # Specify the flat frame + 'skyflat' => '-flat', # Specify the flat frame + 'fringe' => '-fringe', # Specify the fringe frame + 'mask' => '-mask', # Specify the mask frame + 'darkmask' => '-mask', # Specify the mask frame + 'flatmask' => '-mask', # Specify the mask frame +}; + +use constant DELETE_STATS => 0; # Delete the statistics file when done? + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +my $ppStats = can_run('ppStats') or (warn "Can't find ppStats" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +# XXX use PPIMAGE.OUTPUT.RESID for output file (compressed, with pos & neg range allowed) +# my $outputName = $ipprc->filename("PPIMAGE.OUTPUT", $outroot, $class_id); + +my $outputName = $ipprc->filename("PPIMAGE.OUTPUT.RESID", $outroot, $class_id); +my $bin1Name = $ipprc->filename("PPIMAGE.BIN1", $outroot, $class_id); +my $bin2Name = $ipprc->filename("PPIMAGE.BIN2", $outroot, $class_id); +my $outputStats = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id); +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id); + +# Run ppImage & ppStats +unless ($no_op) { + my $command = "$ppImage -file $input_uri $outroot"; + $command .= " -recipe PPIMAGE $recipe"; + $command .= " -recipe PPSTATS RESIDUAL"; + $command .= " -F PPIMAGE.OUTPUT PPIMAGE.OUTPUT.RESID"; + $command .= " -stats $outputStats"; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -threads $threads" if defined $threads; + $command .= " -dbname $dbname" if defined $dbname; + + # Detrend to use in processing + my $detFlag = DETRENDS->{lc($det_type)}; + &my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR) unless defined $detFlag; + $command .= " $detFlag $detrend"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $iter, $exp_id, $class_id, $error_code); + } + &my_die("Couldn't find expected output file: $outputName", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputName); + &my_die("Couldn't find expected output file: $outputStats", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $bin1Name", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($bin1Name); + &my_die("Couldn't find expected output file: $bin2Name", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($bin2Name); + + # Load the raw output stats file + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + # Parse the stats file contents into a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents) or &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + + # Parse the statistics on the residual image + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + + # run ppStats on the binned image + $command = "$ppStats -recipe PPSTATS RESIDUAL $bin2Name"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStats: $error_code", $det_id, $iter, $exp_id, $class_id, $error_code); + } + + # Parse the output contents into a metadata + my $binnedMetadata = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to parse metadata output", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + + # parse the binned image statistics + $binnedStats->parse($binnedMetadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +} + +# Command to update the database +my $command = "$dettool -addresidimfile"; +$command .= " -det_id $det_id"; +$command .= " -iteration $iter"; +$command .= " -ref_det_id $ref_det_id"; +$command .= " -ref_iter $ref_iter"; +$command .= " -exp_id $exp_id"; +$command .= " -class_id $class_id"; +$command .= " -recip $recipe"; +$command .= " -uri $outputName"; +$command .= " -path_base $outroot"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); +$command .= $binnedStats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addresidimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exp_id = shift; # Exposure tag + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and defined $exp_id and not $no_update) { + my $command = "$dettool -addresidimfile"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -ref_det_id $ref_det_id"; + $command .= " -ref_iter $ref_iter"; + $command .= " -exp_id $exp_id"; + $command .= " -class_id $class_id"; + $command .= " -path_base $outroot"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_stack.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_stack.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/detrend_stack.pl (revision 22161) @@ -0,0 +1,287 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $det_id, $iter, $class_id, $det_type, $camera, $outroot, $dbname, $reduction, $threads, $verbose, $save_temps, + $no_update, $no_op, $redirect ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration=s' => \$iter, + 'class_id|i=s' => \$class_id, + 'det_type|t=s' => \$det_type, + 'camera|c=s' => \$camera, + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction=s' => \$reduction, # Reduction class for processing + 'threads=s' => \$threads, + 'verbose' => \$verbose, # Print to stdout + 'save-temps' => \$save_temps, # Save temporary files? + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'redirect-output' => \$redirect, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --class_id --det_type --camera --outroot", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $class_id and + defined $det_type and + defined $camera and + defined $outroot; + +$ipprc->define_camera($camera); +$det_type = uc($det_type); + +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) + or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); + +# optionally redirect the outputs from this script to LOG.IMFILE +$ipprc->redirect_output($logDest) if $redirect; + +# Recipes to use as a function of detrend type +$reduction = "DETREND" unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, $det_type . '_STACK'); # Recipe name to use + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC DETTOOL FLAG + { name => "ROBUST_MEDIAN", type => "clipmean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "clipstdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppMerge = can_run('ppMerge') or (warn "Can't find ppMerge" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# The output file rule name depends on the detrend type +my $FILERULES = { 'FLATMASK' => 'PPMERGE.OUTPUT.MASK', + 'DARKMASK' => 'PPMERGE.OUTPUT.MASK', + 'MASK' => 'PPMERGE.OUTPUT.MASK', + 'BIAS' => 'PPMERGE.OUTPUT.BIAS', + 'DARK' => 'PPMERGE.OUTPUT.DARK', + 'DARK_PREMASK' => 'PPMERGE.OUTPUT.DARK', + 'SHUTTER' => 'PPMERGE.OUTPUT.SHUTTER', + 'FLAT_PREMASK' => 'PPMERGE.OUTPUT.FLAT', + 'DOMEFLAT_PREMASK' => 'PPMERGE.OUTPUT.FLAT', + 'SKYFLAT_PREMASK' => 'PPMERGE.OUTPUT.FLAT', + 'FLAT_RAW' => 'PPMERGE.OUTPUT.FLAT', + 'DOMEFLAT_RAW' => 'PPMERGE.OUTPUT.FLAT', + 'SKYFLAT_RAW' => 'PPMERGE.OUTPUT.FLAT', + 'FLAT' => 'PPMERGE.OUTPUT.FLAT', + 'DOMEFLAT' => 'PPMERGE.OUTPUT.FLAT', + 'SKYFLAT' => 'PPMERGE.OUTPUT.FLAT', + 'FRINGE' => 'PPMERGE.OUTPUT.FRINGE', + }; +my $output_filerule = $FILERULES->{$det_type}; # File rule for output +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless defined $output_filerule; + +# The stats recipe depends on the detrend type +my $STATRECIPES = {'FLATMASK' => 'CHIPSTATS', + 'DARKMASK' => 'CHIPSTATS', + 'MASK' => 'CHIPSTATS', + 'BIAS' => 'CHIPSTATS', + 'DARK' => 'DARKSTATS', + 'DARK_PREMASK' => 'DARKSTATS', + 'SHUTTER' => 'DARKSTATS', + 'FLAT_PREMASK' => 'CHIPSTATS', + 'DOMEFLAT_PREMASK' => 'CHIPSTATS', + 'SKYFLAT_PREMASK' => 'CHIPSTATS', + 'FLAT_RAW' => 'CHIPSTATS', + 'DOMEFLAT_RAW' => 'CHIPSTATS', + 'SKYFLAT_RAW' => 'CHIPSTATS', + 'FLAT' => 'CHIPSTATS', + 'DOMEFLAT' => 'CHIPSTATS', + 'SKYFLAT' => 'CHIPSTATS', + 'FRINGE' => 'CHIPSTATS', + }; +my $statrecipe = $STATRECIPES->{$det_type}; # File rule for output + +# Get list of files to stack +my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + $command = "$dettool -processedimfile -included"; + $command .= " -det_id $det_id"; + $command .= " -class_id $class_id"; + $command .= " -dbname $dbname" if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -processedimfile: $error_code", $det_id, $iter, $class_id, $error_code); + } + + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR); +} + +# Generate MDC file with the inputs +my ($listFile, $listName) = $ipprc->create_temp_file("$outroot.$class_id.list", $save_temps); + +my $num = 0; +foreach my $file (@$files) { + if ($file->{ignored}) { next; } + + print $listFile "INPUT$num\tMETADATA\n"; + $num++; + + my $image = $file->{uri}; # Image name + my $mask = $ipprc->filename( "PPIMAGE.OUTPUT.MASK", $file->{path_base}, $class_id ); # Mask name + my $weight = $ipprc->filename( "PPIMAGE.OUTPUT.WEIGHT", $file->{path_base}, $class_id ); # Weight name + + &my_die("Image $image does not exist", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $image ); + print $listFile "\tIMAGE\tSTR\t" . $image . "\n"; + + if ($ipprc->file_exists( $mask )) { + print $listFile "\tMASK\tSTR\t" . $mask . "\n"; + } + if ($ipprc->file_exists( $weight )) { + print $listFile "\tWEIGHT\tSTR\t" . $weight . "\n"; + } + + print $listFile "END\n\n"; +} +close $listFile; + + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $outputStack = $ipprc->filename($output_filerule, $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Output name +my $outputCount = $ipprc->filename("PPMERGE.OUTPUT.COUNT", $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Count image +my $outputSigma = $ipprc->filename("PPMERGE.OUTPUT.SIGMA", $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Stdev image +my $outputStats = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Statistics name +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Trace messages + +$command = "$ppMerge $listName $outroot"; # Command to run +$command .= " -recipe PPMERGE $recipe"; +$command .= ' -type ' . uc($det_type); # Type of stacking to perform +$command .= " -stats $outputStats"; # Statistics output filename +$command .= " -recipe PPSTATS $statrecipe"; +$command .= " -tracedest $traceDest -log $logDest"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= " -threads $threads" if defined $threads; + +# Stack the files +unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppMerge: $error_code", $det_id, $iter, $class_id, $error_code); + } + &my_die("Unable to find expected output file: $outputStack\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStack); + &my_die("Unable to find expected output file: $outputCount\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputCount); + &my_die("Unable to find expected output file: $outputSigma\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputSigma); + &my_die("Unable to find expected output file: $outputStats\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats); + + # Get the statistics on the stacked image + open(my $statsFile, $ipprc->file_resolve("$outputStats")) or + &my_die("Can't open statistics file $outputStats: $!", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); + my $contents = do { local $/; <$statsFile> }; # Contents of file + close($statsFile); + + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse($contents) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); + + $stats->parse($metadata) or + &my_die("Unable to find all values in statistics output.", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); +} + +# Command to update the database +$command = "$dettool -addstacked"; +$command .= " -det_id $det_id"; +$command .= " -iteration $iter"; +$command .= " -class_id $class_id"; +$command .= " -uri $outputStack"; +$command .= " -recip $recipe"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the resultant into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addstacked: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and defined $class_id and not $no_update) { + my $command = "$dettool -addstacked"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -class_id $class_id"; + # XXX EAM : we should add this to the db : $command .= " -path_base $outroot"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/diff_skycell.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/diff_skycell.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/diff_skycell.pl (revision 22161) @@ -0,0 +1,275 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($diff_id, $dbname, $threads, $outroot, $verbose, $no_update, $no_op, $redirect); +GetOptions( + 'diff_id|d=s' => \$diff_id, # Diff identifier + 'dbname|d=s' => \$dbname, # Database name + 'threads=s' => \$threads, # Number of threads to use + 'outroot=s' => \$outroot, # Output root name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --diff_id --outroot", + -exitval => 3, + ) unless defined $diff_id + and defined $outroot; + +# XXX camera is not known here; cannot use filerules... +# my $logDest = $ipprc->filename("LOG.EXP", $outroot); + +my $logDest = "$outroot.log"; +$ipprc->redirect_output($logDest) if $redirect; + +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC DIFFTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "DT_SUB", type => "sum", flag => "-dtime_diff", dtype => "float" }, + { name => "SUBTRACTION.NUM", type => "mean", flag => "-stamps_num", dtype => "int" }, + { name => "SUBTRACTION.RMS", type => "mean", flag => "-stamps_rms", dtype => "float" }, + { name => "NUM_SOURCES", type => "sum", flag => "-sources", dtype => "int" }, + { name => "GOOD_PIXEL_FRAC", type => "mean", flag => "-good_frac", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $ppSub = can_run('ppSub') or (warn "Can't find ppSub" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of components for subtraction +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $files; +{ + my $command = "$difftool -inputskyfile -diff_id $diff_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool -inputskyfile: $error_code", $diff_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $diff_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $diff_id, $PS_EXIT_PROG_ERROR); +} + +&my_die("Subtraction list does not contain exactly two elements", $diff_id, $PS_EXIT_SYS_ERROR) unless + scalar @$files == 2; + +# Identify the input and the template +my ($input, $inputMask, $inputWeight, $inputPath); # Input files and path +my ($template, $templateMask, $templateWeight, $templatePath, $templateSources); # Template files and path +my $tess_id; # Tesselation identifier +my $skycell_id; # Skycell identifier +my $camera; # Camera +foreach my $file (@$files) { + if (defined $file->{template} and $file->{template}) { + $template = $file->{uri}; + $templatePath = $file->{path_base}; + if ($file->{warp_id} == 0) { + $templateMask = "PPSTACK.OUTPUT.MASK"; + $templateWeight = "PPSTACK.OUTPUT.WEIGHT"; + $templateSources = "PSPHOT.OUT.CMF.MEF"; ## this must be consistent with the value in stack_skycell.pl:161 + ## use an explicit stack name for psphot output objects + } else { + $templateMask = "PSWARP.OUTPUT.MASK"; + $templateWeight = "PSWARP.OUTPUT.WEIGHT"; + $templateSources = "PSWARP.OUTPUT.SOURCES"; + } + } else { + $input = $file->{uri}; + $inputPath = $file->{path_base}; + if ($file->{warp_id} == 0) { + $inputMask = "PPSTACK.OUTPUT.MASK"; + $inputWeight = "PPSTACK.OUTPUT.WEIGHT"; + } else { + $inputMask = "PSWARP.OUTPUT.MASK"; + $inputWeight = "PSWARP.OUTPUT.WEIGHT"; + } + } + if (defined $tess_id) { + &my_die("Tesselation identifiers don't match", $diff_id, $PS_EXIT_SYS_ERROR) unless + $file->{tess_id} eq $tess_id; + } else { + $tess_id = $file->{tess_id}; + } + if (defined $skycell_id) { + &my_die("Skycell identifiers don't match", $diff_id, $PS_EXIT_SYS_ERROR) unless + $file->{skycell_id} eq $skycell_id; + } else { + $skycell_id = $file->{skycell_id}; + } + if (defined $camera) { + &my_die("Cameras don't match", $diff_id, $PS_EXIT_SYS_ERROR) unless $file->{camera} eq $camera; + } else { + $camera = $file->{camera}; + } + +} + +&my_die("Unable to identify template", $diff_id, $PS_EXIT_SYS_ERROR) unless defined $template; +&my_die("Unable to identify input", $diff_id, $PS_EXIT_SYS_ERROR) unless defined $input; +&my_die("Unable to identify camera", $diff_id, $PS_EXIT_SYS_ERROR) unless defined $camera; +$ipprc->define_camera($camera); + +# print "templateMask: $templateMask\n"; +# print "templatePath: $templatePath\n"; +# print "inputMask: $inputMask\n"; +# print "inputPath: $inputPath\n"; +# print "templateWeight: $templateWeight\n"; +# print "inputWeight: $inputWeight\n"; +# print "templateSources: $templateSources\n"; + +$templateMask = $ipprc->filename($templateMask, $templatePath); +$inputMask = $ipprc->filename($inputMask, $inputPath); +$templateWeight = $ipprc->filename($templateWeight, $templatePath); +$inputWeight = $ipprc->filename($inputWeight, $inputPath); +$templateSources = $ipprc->filename($templateSources, $templatePath); + +print "templateMask: $templateMask\n"; +print "templatePath: $templatePath\n"; +print "inputMask: $inputMask\n"; +print "inputPath: $inputPath\n"; +print "templateWeight: $templateWeight\n"; +print "inputWeight: $inputWeight\n"; +print "templateSources: $templateSources\n"; + +&my_die("Couldn't find input: $template", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($template); +&my_die("Couldn't find input: $templateMask", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($templateMask); +&my_die("Couldn't find input: $templateWeight", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($templateWeight); +&my_die("Couldn't find input: $input", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input); +&my_die("Couldn't find input: $inputMask", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputMask); +&my_die("Couldn't find input: $inputWeight", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputWeight); +&my_die("Couldn't find input: $templateSources", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($templateSources); + +# Get the output filenames +my $outputName = $ipprc->filename("PPSUB.OUTPUT", $outroot); +my $outputMask = $ipprc->filename("PPSUB.OUTPUT.MASK", $outroot); +my $outputWeight = $ipprc->filename("PPSUB.OUTPUT.WEIGHT", $outroot); +my $configuration = $ipprc->filename("PPSUB.CONFIG", $outroot); +my $outputSources = $ipprc->filename("PSPHOT.OUT.CMF.MEF", $outroot); +#my $bin1Name = $ipprc->filename("PPSUB.BIN1", $outroot); +#my $bin2Name = $ipprc->filename("PPSUB.BIN2", $outroot); +my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); + +# Perform subtraction +unless ($no_op) { + my $command = "$ppSub $input $template $outroot"; + $command .= " -inmask $inputMask"; + $command .= " -refmask $templateMask"; + $command .= " -inweight $inputWeight"; + $command .= " -refweight $templateWeight"; + $command .= " -stats $outputStats"; + $command .= " -threads $threads" if defined $threads; + $command .= " -recipe PPSTATS WARPSTATS"; + $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; + $command .= " -sources $templateSources"; + $command .= " -photometry"; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dumpconfig $configuration"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppSub: $error_code", $diff_id, $error_code); + } + &my_die("Couldn't find expected output file: $outputName", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); + &my_die("Couldn't find expected output file: $outputMask", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + &my_die("Couldn't find expected output file: $outputWeight", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); + &my_die("Couldn't find expected output file: $outputSources", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); +# &my_die("Couldn't find expected output file: $bin1Name", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin1Name); +# &my_die("Couldn't find expected output file: $bin2Name", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin2Name); + &my_die("Couldn't find expected output file: $outputStats", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); + + # Get the statistics on the residual image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $diff_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + my $metadata = $mdcParser->parse(join "", @contents) or + &my_die("Unable to parse metadata config doc", $diff_id, $PS_EXIT_PROG_ERROR); + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $diff_id, $PS_EXIT_PROG_ERROR); +} + +unless ($no_update) { + + # Add the subtraction result + { + my $command = "$difftool -adddiffskyfile -diff_id $diff_id -uri $outputName -path_base $outroot"; + $command .= $stats->cmdflags(); + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool -adddiffskyfile: $error_code", $diff_id, $error_code); + } + } +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $diff_id = shift; # Diff identifier + my $exit_code = shift; # Exit code to add + + warn($msg); + if (defined $diff_id and not $no_update) { + my $command = "$difftool -adddiffskyfile -diff_id $diff_id -code $exit_code"; + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + run(command => $command, verbose => $verbose); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ds9_cmf_regions.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ds9_cmf_regions.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ds9_cmf_regions.pl (revision 22161) @@ -0,0 +1,134 @@ +#!/usr/bin/env perl + +use strict; +use Astro::FITS::CFITSIO qw( :constants ); +use File::Temp qw( tempfile ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use Data::Dumper; +use Carp; + +Astro::FITS::CFITSIO::PerlyUnpacking(1); + +my $xpaset = `which xpaset`; +my $xpaget = `which xpaget`; + +die "Unable to find xpaget and xpaset.\n" unless ($xpaset =~ /\S+/ and $xpaget =~ /\S+/); + + +my ( $filename, # Filename containing photometry + $extname, # Extension name containing photometry + $frame, # Frame number in ds9 + $colour, # Region colour + $flag_colour, # Flagged source region colour + $flag, # Flags + $radius, # Radius for circle + $save_temps + ); + +# Defaults +$colour = "blue"; +$flag_colour = "red"; +$radius = 5; +$flag = 0x3888; + +GetOptions( + 'file=s' => \$filename, + 'ext=s' => \$extname, + 'frame=s' => \$frame, + 'colour=s' => \$colour, + 'flag-colour=s' => \$flag_colour, + 'flag=o' => \$flag, + 'radius=f' => \$radius, + 'save-temps' => \$save_temps, # Save temporary files? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --file --ext", + -exitval => 3) + unless defined $filename + and defined $extname; + +my $status; # Status of FITSIO calls +my $fits = Astro::FITS::CFITSIO::open_file( $filename, READONLY, $status ); # FITS file handle +check_fitsio($status); +$fits->movnam_hdu(BINARY_TBL, $extname, 0, $status) and check_fitsio($status); +my $numRows; # Number of rows in table +$fits->get_num_rows($numRows, $status) and check_fitsio($status); + +my ($xCol, $yCol, $flagCol); # Column numbers for x,y, flag +$fits->get_colnum(0, 'X_PSF', $xCol, $status) and check_fitsio($status); +$fits->get_colnum(0, 'Y_PSF', $yCol, $status) and check_fitsio($status); +$fits->get_colnum(0, 'FLAGS', $flagCol, $status) and check_fitsio($status); + +my ($x, $y, $flags); # Coordinates and flags read from table +$fits->read_col(TFLOAT, $xCol, 1, 1, $numRows, 0, $x, undef, $status) and check_fitsio($status); +$fits->read_col(TFLOAT, $yCol, 1, 1, $numRows, 0, $y, undef, $status) and check_fitsio($status); +$fits->read_col(TINT, $flagCol, 1, 1, $numRows, 0, $flags, undef, $status) and check_fitsio($status); +$fits->close_file($status); + +my ($coordFile, $coordName) = tempfile( "/tmp/ds9_cmf_regions.XXXX", UNLINK => !$save_temps ); +for (my $i = 0; $i < $numRows; $i++) { + print $coordFile "image; circle(" . ($$x[$i] + 1) . ',' . ($$y[$i] + 1) . ",$radius) \# color = " . + (($$flags[$i] & $flag) ? $flag_colour : $colour) . "\n"; +} +close $coordFile; + +my @settings = settings_save("regions format", + "regions system"); # Settings to save + +xpaset("frame $frame") if defined $frame; +xpaset("regions format ds9"); +xpaset("regions system image"); +xpaset("regions load $coordName"); + +xpaset(@settings); + +print "Plotted $numRows sources.\n"; + +### Pau. + + + + +# From Astro::FITS::CFITSIO demo +sub check_fitsio +{ + my $status = shift; # Status of FITSIO calls + + if ($status != 0) { + my $msg; # Message to output + Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg ); + croak "CFITSIO error: $msg\n"; + } +} + +# Save specified settings +sub settings_save +{ + my @settings; # Values of settings + foreach my $setting (@_) { + my @values = xpaget($setting); + push @settings, $setting . ' ' . shift @values; + } + return @settings; +} + + +# XPA subroutines courtesy Derek Fox +sub xpaset { + foreach my $cmd (@_) { + system("xpaset -p ds9 $cmd"); + } +} +sub xpaget { + my @out; + foreach my $cmd (@_) { + my $output = `xpaget ds9 $cmd`; + push @out, $output; + } + return @out; +} + + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/fake_imfile.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/fake_imfile.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/fake_imfile.pl (revision 22161) @@ -0,0 +1,241 @@ +#!/usr/bin/env perl + +## this script is run on every imfile to perform the fake source +## analysis and the forced photometry analysis. + +## For the fake source analysis, we load the image, inject a number of +## fake source (with a flat mag distribution), save the input source, +## then perform photometry on the image to recover the input sources. +## we should save a file with the matched input and recovered +## sources. the output metadata should include an analysis of the +## recovery rate and measurement error as a function of magnitude + +## For the forced photometry, we need to use the measured astrometry +## to select the desired locations in pixel coord. + +# XXX : seeing needs to be determined from the input PSF (not currently done in ppSim) +# ppSim output -input input.fits -cmf input.cmf -psf input.psf -seeing 0.563760 -recipe PPSIM FAKEPHOT + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ( $exp_id, $fake_id, $class_id, $chiproot, $camroot, $camera, $outroot, + $dbname, $reduction, $verbose, $no_update, $no_op, $redirect ); +GetOptions( + 'exp_id=s' => \$exp_id, # Exposure identifier + 'fake_id=s' => \$fake_id, # Chiptool identifier + 'class_id=s' => \$class_id, # Class identifier + 'chiproot=s' => \$chiproot, # Input Chip files (root) + 'camroot=s' => \$camroot, # Input Camera files (root) + 'camera|c=s' => \$camera, # Camera + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --exp_id --fake_id --class_id --chiproot --camroot --camera --outroot", + -exitval => 3) unless + defined $exp_id and + defined $fake_id and + defined $class_id and + defined $chiproot and + defined $camroot and + defined $camera and + defined $outroot; + +$ipprc->define_camera($camera); + +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR); + +$ipprc->redirect_output($logDest) if $redirect; + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, 'FAKEPHOT'); # Recipe to use +unless ($recipe) { + &my_die("Couldn't find selected reduction for FAKE: $reduction\n", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR); +} + +# values to extract from output metadata and the stats to calculate +# XXX not sure if this is needed by fake_imfile.pl +# my $STATS = +# [ +# # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG +# { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, +# { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, +# { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, +# ]; +# my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $faketool = can_run('faketool') or (warn "Can't find faketool" and $missing_tools = 1); +my $ppSim = can_run('ppSim') or (warn "Can't find ppSim" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +## these names are used in ppImage, and thus may be URIs +my $chipImage = $ipprc->filename("PPIMAGE.CHIP", $chiproot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $chipMask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chiproot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $chipWeight = $ipprc->filename("PPIMAGE.CHIP.WEIGHT", $chiproot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $chipPSF = $ipprc->filename("PSPHOT.PSF.SAVE", $chiproot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $cameraObjects = $ipprc->filename("PSASTRO.OUTPUT", $camroot) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR); + +# XXX check for existence of input data +# &my_die("Couldn't find input file: $uri\n", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri); + +## get the PPSIM recipe for this camera and FAKEPHOT reduction +my $command = "$ppConfigDump -camera $camera -dump-recipe PPSIM -recipe PPSIM $recipe -"; +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR); +} +my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR); + +## allow the output images to be optional, depending on the recipe / reduction class +my $skipFake = metadataLookupBool($recipeData, 'SKIP.FAKE'); + +# Run ppSim +unless ($no_op || $skipFake) { + # examine the PPSIM recipe to decide if we need to run this or opt out + + my $command = "$ppSim $outroot"; + $command .= " -input $chipImage"; + # XXX add input mask and weight to ppSim + # $command .= " -mask $chipMask"; + # $command .= " -weight $chipWeight"; + $command .= " -cmf $cameraObjects"; + $command .= " -psf $chipPSF"; + $command .= " -recipe PPSIM $recipe"; + $command .= " -dbname $dbname" if defined $dbname; + $command .= " -tracedest $traceDest"; + $command .= " -log $logDest"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppSim: $error_code", $exp_id, $fake_id, $class_id, $error_code); + } + + # XXX check for output files? + # &my_die("Couldn't find expected output file: $outputBin1\n", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1); + + # XXX use this to parse the output metadata : eg, detection limits, Nfakes, Nforced + # Get the statistics on the processed image + # my $statsFile; # File handle + # open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR); + # my @contents = <$statsFile>; # Contents of file + # close $statsFile; + # + # # parse the statistics MDC file + # my $mdcParser = PS::IPP::Metadata::Config->new(); # Parser for metadata config files + # my $metadata = $mdcParser->parse(join "", @contents); + # unless ($metadata) { + # &my_die("Unable to parse metadata config doc", $exp_id, $fake_id, $class_id, $PS_EXIT_PROG_ERROR); + # } + # + # # extract the stats from the metadata + # unless ($stats->parse($metadata)) { + # &my_die("Failure extracting metadata from the statistics output file.\n", $exp_id, $fake_id, $class_id, $PS_EXIT_PROG_ERROR); + # } +} else { + print "skipping ppSim processing\n"; +} + +# command to update database +$command = "$faketool -addprocessedimfile"; +$command .= " -fake_id $fake_id"; +$command .= " -exp_id $exp_id"; +$command .= " -class_id $class_id"; +$command .= " -path_base $outroot"; +$command .= " -dbname $dbname" if defined $dbname; +# XXX add this after defined +# $command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform faketool -addprocessedimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_id = shift; # rawExp identifier + my $fake_id = shift; # fakeRun identifier + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $exp_id and defined $fake_id and defined $class_id and not $no_update) { + my $command = "$faketool -addprocessedimfile"; + $command .= " -fake_id $fake_id"; + $command .= " -exp_id $exp_id"; + $command .= " -class_id $class_id"; + $command .= " -path_base $outroot"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/flatcorr_init.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/flatcorr_init.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/flatcorr_init.pl (revision 22161) @@ -0,0 +1,212 @@ +#!/usr/bin/env perl + +## USAGE:flatcorr_init.pl +## given dbname, dvodb, filter, time range, camera, telescope, etc? +## select matching images and register as a new flatcorr run + +## should this be implemented as a ippTool program? +## this is a lot like dettool -definebyquery +## flatcorr -definebyquery +## this should queue the selected images with a mode that stops at camera +## ?? how do we block this batch from running through warp ?? + +## we also need the test / query tool: +## flatcorr -pendingcorr +## examine the chip & camera stages and check for all images to have completed (successfully or not) + +## allow a certain fraction of failures... + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Storable qw(freeze thaw); +use File::Basename qw( basename); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dvo_id, $catdir, $region, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'dvo_id|i=s' => \$dvo_id, + 'catdir|c=s' => \$catdir, + 'region|r=s' => \$region, + 'dbname|d=s' => \$dbname,# Database name + 'workdir|w=s' => \$workdir, # Working directory for output files + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --dvo_id --catdir --region", + -exitval => 3) unless + defined $dvo_id and + defined $catdir and + defined $region; + +# Look for programs we need +my $missing_tools; +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $uniphot = can_run('uniphot') or (warn "Can't find uniphot" and $missing_tools = 1); +my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); +my $caltool = can_run('caltool') or (warn "Can't find caltool" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# select the primary filters from DVO query? +my (@filters) = `photcodeList -average`; + +# parse the region (RAs,RAe:DECs,DECe) : item = +/-NNN.NNNN +my @coords = split (":", $region); +my ($RAs, $RAe) = split (",", $coords[0]); +my ($DECs, $DECe) = split (",", $coords[1]); + +# Run addstar -resort +{ + my $command = "$addstar -resort"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RESORT", $status, $dbname); + } +} + +# Run relphot (filter) for each filter +{ + foreach my $filter (@filters) { + my $command = "$relphot $filter"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELPHOT", $status, $dbname); + } + } +} + +# Run uniphot (filter) for each filter +# XXX skip this one? run less frequently? +if (0) { + foreach my $filter (@filters) { + my $command = "$uniphot $filter"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "UNIPHOT", $status, $dbname); + } + } +} + +{ + my $command = "$relastro -objects"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.OBJECTS", $status, $dbname); + } +} + +{ + my $command = "$relastro -images"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.IMAGES", $status, $dbname); + } +} + +my $command = "$caltool -addcalrun"; +$command .= " -dvo_id $dvo_id"; +$command .= " -region $region"; +$command .= " -last_step RELASTRO.IMAGES"; +$command .= " -status SUCCESS"; +$command .= " -dbname $dbname" if defined $dbname; + +# Push the results into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $dvo_id = shift; + my $region = shift; + my $last_step = shift; + my $status = shift; + my $dbname = shift; + + carp($msg); + if (defined $dvo_id && defined $region && defined $last_step && defined $status and not $no_update) { + my $command = "$caltool -addcalrun"; + $command .= " -dvo_id $dvo_id"; + $command .= " -region $region"; + $command .= " -last_step $last_step"; + $command .= " -status $status"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Pau. + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/flatcorr_proc.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/flatcorr_proc.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/flatcorr_proc.pl (revision 22161) @@ -0,0 +1,324 @@ +#!/usr/bin/env perl + +## USAGE:flatcorr_proc.pl --dbname --corr_id + +## this script does the following steps: + +# extract the details of the flatcorr run: dvodb, filter, camera, etc? + +# relphot -D CATDIR $dvodb -grid (outgrid.fits) (filter) -region 0 360 -90 90 (other parameters?) + +# dvoMakeCorr -file outgrid.fits -ref ref.fits outcorr + +# dettool -register -det_type FLATCORR -filelevel (level) -workdir -inst, etc + +# foreach $imfile () +# dettool -register_imfile -uri, etc, etc + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Storable qw(freeze thaw); +use File::Basename qw( basename); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($corr_id, $dvodb, $region, $filter, $dbname, $workdir, $verbose, $no_update, $no_op); +GetOptions( + 'corr_id|i=s' => \$corr_id, + 'dvodb|c=s' => \$dvodb, + 'region|r=s' => \$region, + 'filter|f=s' => \$filter, + 'dbname|d=s' => \$dbname,# Database name + 'workdir|w=s' => \$workdir, # Working directory for output files + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --corr_id --dvodb --region --filter --workdir", + -exitval => 3) unless + defined $corr_id and + defined $dvodb and + defined $region and + defined $workdir and + defined $filter; + +# Look for programs we need +my $missing_tools; +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +my $dvoMakeCorr = can_run('dvoMakeCorr') or (warn "Can't find dvoMakeCorr" and $missing_tools = 1); +my $detselect = can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1); +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $flatcorr = can_run('flatcorr') or (warn "Can't find flatcorr" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +my $outgrid = "$workdir/grid.$corr_id.fits"; +my $outcorr = "$workdir/corr.$corr_id"; + +# parse the region (RAs,RAe:DECs,DECe) : item = +/-NNN.NNNN +my @coords = split (":", $region); +my ($RAs, $RAe) = split (",", $coords[0]); +my ($DECs, $DECe) = split (",", $coords[1]); + +# Run addstar -resort to ensure the db is indexed +{ + my $command = "echo $addstar -resort"; + $command .= " -D CATDIR $dvodb"; + $command .= " -region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort for dvodb $dvodb on region $region: $error_code", $corr_id, $error_code); + } +} + +# Run relphot (filter) for the specified region (need to clarify the options like imfreeze) +{ + my $command = "echo $relphot $filter"; + $command .= " -D CATDIR $dvodb"; + $command .= " -region $RAs $RAe $DECs $DECe"; + $command .= " -grid $outgrid"; + $command .= " -imfreeze"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform relphot -grid for dvodb $dvodb on region $region: $error_code", $corr_id, $error_code); + } +} + +# get a single input exposure +# flatcorr -inputexp -corr_id $corr_id -limit 1 +my $chip_id; +{ + my $command = "$flatcorr -inputexp"; + $command .= " -corr_id $corr_id"; + $command .= " -limit 1"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform camtool: $error_code", $corr_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $corr_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + my $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $corr_id, $PS_EXIT_PROG_ERROR); + + # check for existence + my $file = $$files[0]; + $chip_id = $file->{chip_id}; +} + +# get the list of imfiles for the single input exposure +# flatcorr -inputimfile -chip_id $chip_id +my $files; +{ + my $command = "$flatcorr -inputimfile"; + $command .= " -chip_id $chip_id"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform camtool: $error_code", $corr_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $corr_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $corr_id, $PS_EXIT_PROG_ERROR); +} + +# set up the detrend run to store the corrected imfiles +my $det_id; +{ + # get the filelevel from one of these chips + my $file = $$files[0]; + + my $filelevel = $file->{filelevel}; + my $camera = $file->{camera}; + my $telescope = $file->{telescope}; + my $filter = $file->{filter}; + + ## XXX the flat-field correction is only defined for a single + ## camera. we are grabbing the camera here, but we are assuming + ## the flatcorrRun was defined for only a single camera. + $ipprc->define_camera($camera); + + my $command = "$dettool -register_detrend"; + $command .= " -det_type FLATCORR"; + $command .= " -filelevel $filelevel"; + $command .= " -workdir $workdir"; + $command .= " -inst $camera"; + $command .= " -telescope $telescope"; + $command .= " -filter $filter"; + $command .= " -dbname $dbname" if defined $dbname; + + ## the flat-field correction is valid for any airmass, exptime, solangle + ## for now, we assume the posangle and ccd_temp are not important + ## XXX someone needs to set the use_begin, use_end values?? + ## XXX inherit a label from the flatcorrRun? + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to register new detrend: $error_code", $corr_id, $PS_EXIT_PROG_ERROR); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $corr_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + my $output = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $corr_id, $PS_EXIT_PROG_ERROR); + + # $file = $$output[0]; + $det_id = $$output[0]->{det_id}; +} + +# use input chip image as a reference image +foreach my $file (@$files) { + # create the detrend correction for the imfiles based on the input imfiles + my $reffile = $file->{uri}; + my $class_id = $file->{class_id}; + + my $uri = $ipprc->filename("DVOCORR.OUTPUT", $outcorr, $class_id); + unless ($uri) { + &my_die ("Unable to find DVOCORR.OUTPUT in filerules", $corr_id, $PS_EXIT_PROG_ERROR); + } + + my $command = "echo $dvoMakeCorr $outcorr"; + $command .= " -file $outgrid"; + $command .= " -ref $reffile"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform dvoMakeCorr: $error_code", $corr_id, $PS_EXIT_PROG_ERROR); + } + + # register the detrend correction imfile + $command = "$dettool -register_detrend_imfile"; + $command .= " -det_id $det_id"; + $command .= " -class_id $class_id"; + $command .= " -uri $uri"; + $command .= " -dbname $dbname" if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to register new detrend: $error_code", $corr_id, $PS_EXIT_PROG_ERROR); + } +} + +# set the detrun state to 'stop' +{ + my $command = "$dettool -updatedetrun"; + $command .= " -det_id $det_id"; + $command .= " -state stop"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform dettool -updatedetrun: $error_code"); + exit($error_code); + } +} + +# Push the results into the database +{ + my $command = "$flatcorr -addprocess"; + $command .= " -corr_id $corr_id"; + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_update) { + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } + } else { + print "skipping command: $command\n"; + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $corr_id = shift; + my $exit_code = shift; + + carp($msg); + if (not $no_update) { + my $command = "$flatcorr -addprocess"; + $command .= " -corr_id $corr_id"; + $command .= " -code $exit_code"; + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Pau. + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/gpc_seeing.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/gpc_seeing.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/gpc_seeing.pl (revision 22161) @@ -0,0 +1,28 @@ +#!/usr/bin/env perl +# basic ISP transmission analysis: + +if (@ARGV != 1) { die "USAGE: gpc_seeing.pl (input.fits)\n"; } +$input = $ARGV[0]; + +# for input file /path/foo.fits, use /path/foo for output + +@words = split ('\.', $input); +if (@words > 1) { pop @words; } +$output = join (".", @words); + +# use constant RECIPE => 'PPIMAGE_OBDSFRA'; # Recipe to use +$RECIPE_PPIMAGE = 'PPIMAGE_OP'; +$RECIPE_PSPHOT = 'PSPHOT.SEEING'; + +# recommend only processing to PSFMODEL +vsystem ("ppImage -file $input $output -recipe PPIMAGE $RECIPE_PPIMAGE -recipe PSPHOT $RECIPE_PSPHOT"); +if ($status) { die "failure running ppImage\n"; } + +# XXX otis can read the output psf model, or we can supply a program to interpret the model + +sub vsystem { + print STDERR "@_\n"; + my $status = system ("@_"); + $status; +} + Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_cleanup.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_cleanup.pl (revision 22161) @@ -0,0 +1,374 @@ +#!/usr/bin/env perl + +# this script is used to cleanup the files from the different ipp +# stages. It can be called with one of two cleanup modes: clean and +# purge. the former removes temporary data files, leaving behind +# enough information for the results to be rebuilt. The latter +# removes all but basic logging data. + +use warnings; +use strict; +use Carp; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use PS::IPP::Config 1.01 qw( :standard ); +use PS::IPP::Metadata::List qw( parse_md_list ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $ipprc = PS::IPP::Config->new(); # this is used for PATH, NEB filename conversions + +# Parse the command-line arguments +my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist); +GetOptions('stage=s' => \$stage, # which analysis stage to clean? + 'camera|i=s' => \$camera, # user-supplied camera name + 'stage_id=s' => \$stage_id, # id for this stage (only needed for certain stages) + 'mode|m=s' => \$mode, # cleanup mode (clean / purge) + 'path_base=s' => \$path_base, # basename for files + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # pretend but don't actually inject + 'helplist' => \$helplist # give help listing + ) or pod2usage( 2 ); + +pod2usage( -msg => "remove temporary / all data files for an IPP analysis stage", + -exitval => 2) if defined $helplist; + +pod2usage( -msg => "Usage: $0 --camera (name) --stage (stage) --stage_id (stage_id) --mode (mode) [--path_base (path)] [--dbname dbname] [--no-op] [--help]", + -exitval => 2 ) if scalar @ARGV; + +pod2usage( -msg => "Required options:--camera (name) --stage (stage) --mode (mode)", + -exitval => 3) unless + defined $camera and + defined $stage and + defined $mode; + +# $mode must be one of "goto_cleaned" or "goto_purged" +unless (($mode eq "goto_cleaned") || ($mode eq "goto_purged")) { + die "invalid cleanup mode $mode\n"; +} + +my %stages = ( chip => 1, camera => 1, fake => 1, warp => 1, stack => 1, diff => 1); +unless ($stages{$stage}) { + die "unknown stage $stage for ipp_cleanup.pl\n"; +} + +$ipprc->define_camera($camera); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# choice of files to delete depends on the stage +if ($stage eq "chip") { + + die "--stage_id required for stage chip\n" if !$stage_id; + ### select the imfiles for this entry + + # this stage uses 'chiptool' + my $chiptool = can_run('chiptool') or die "Can't find chiptool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $imfiles; # Array of component files + my $command = "$chiptool -pendingcleanupimfile -chip_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "chip", $stage_id, $PS_EXIT_PROG_ERROR); + + # loop over all of the imfiles, determine the path_base and class_id for each + foreach my $imfile (@$imfiles) { + my $class_id = $imfile->{class_id}; + my $path_base = $imfile->{path_base}; + my $status = 1; + + # don't clean up unless the data needed to update is available + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for chipRun $stage_id $class_id " + . " because config file is missing\n"; + $status = 0; + } + } + + if ($status) { + # array of actual filenames to delete + my @files = (); + + # delete the temporary image datafiles + addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.OUTPUT.WEIGHT", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.CHIP.WEIGHT", $path_base, $class_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.JPEG", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id); + } + + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$chiptool -chip_id $stage_id -class_id $class_id"; + if ($mode eq "goto_purged") { + $command .= " -topurgedimfile"; + } else { + $command .= " -tocleanedimfile"; + } + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); + } + } else { + my $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $class_id -code 1"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); + } + } + } + +} elsif ($stage eq "camera") { + die "--stage_id required for stage camera\n" if !$stage_id; + # this stage uses 'camtool' + my $camtool = can_run('camtool') or die "Can't find camtool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $exps; # Array of component files + my $command = "$camtool -pendingcleanupexp -cam_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "camera", $stage_id, $PS_EXIT_PROG_ERROR); + + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "camera", $stage_id, $PS_EXIT_PROG_ERROR); + + my $n_exps = @$exps; + &my_die("unexpected number of exposures $n_exps", "camera", $stage_id, $PS_EXIT_PROG_ERROR) + if $n_exps != 1; + + my $exp = $exps->[0]; + my $path_base = $exp->{path_base}; + + my $status = 1; + # don't clean up unless the data needed to update is available + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for camRun $stage_id because config file is missing\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + # delete the temporary image datafiles + addFilename (\@files, "PSASTRO.OUTPUT", $path_base); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.JPEG1", $path_base); + addFilename (\@files, "PPIMAGE.JPEG2", $path_base); + addFilename (\@files, "PSASTRO.STATS", $path_base); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$camtool -cam_id $stage_id -updaterun"; + if ($mode eq "goto_cleaned") { + $command .= " -state cleaned"; + } else { + $command .= " -state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code); + } + } else { + my $command = "$camtool -updateprocessedexp -cam_id $stage_id -code 1"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + exit 0; +} elsif ($stage eq "warp") { + die "--stage_id required for stage warp\n" if !$stage_id; + # this stage uses 'warptool' + my $warptool = can_run('warptool') or die "Can't find warptool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $skyfiles; # Array of component files + my $command = "$warptool -pendingcleanupskyfile -warp_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "warp", $stage_id, $PS_EXIT_PROG_ERROR); + + $skyfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "warp", $stage_id, $PS_EXIT_PROG_ERROR); + + my @files = (); + foreach my $skyfile (@$skyfiles) { + my $path_base = $skyfile->{path_base}; + my $skycell_id = $skyfile->{skycell_id}; + + my $status = 1; + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" . + " because config file is missing\n"; + $status = 0; + } + } + if ($status) { + # delete the temporary image datafiles + addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id ); + addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); + addFilename(\@files, "PSWARP.OUTPUT.WEIGHT", $path_base, $skycell_id); + addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id); + + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename(\@files, "PSWARP.BIN1", $path_base, $skycell_id ); + addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id ); + addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id ); + # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base); + + # XXX: do we want to delete these? + # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id); + # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$warptool -warp_id $stage_id -skycell_id $skycell_id"; + if ($mode eq "goto_purged") { + $command .= " -topurgedskyfile"; + } else { + $command .= " -tocleanedskyfile"; + } + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); + } + } else { + # XXX: -updateskyfile mode does not exist, need to add it + my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -code 1"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + exit 0; +} + +# left TODO +# fake : faketool : -pendingcleanupimfile (loop over imfiles) +# stack: stacktool : -pendingcleanupskyfile (loop over skyfiles) +# diff: difftool : -pendingcleanupskyfile + +die "ipp_cleanup.pl -stage $stage not yet implemented\n"; + +sub delete_files +{ + my $files = shift; # reference to a list of files to unlink + + # this script is, of course, very dangerous. + foreach my $file (@$files) { + print STDERR "unlinking $file\n"; + # XXX: need to handle files in nebulous + unlink $file; + } + return 1; +} + +sub addFilename +{ + my $files = shift; # reference to a list of files to unlink + my $filerule = shift; # filerule to add + my $path_base = shift; # base filename + my $class_id = shift; # class_id, if needed + + my $file = $ipprc->filename($filerule, $path_base, $class_id); + + push @$files, $file; + return 1; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $stage = shift; # stage name + my $stage_id = shift; # identifier + my $exit_code = shift; # Exit code + # outputImage and path_base are globals + + carp($msg); + exit $exit_code; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_darkstats.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_darkstats.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_darkstats.pl (revision 22161) @@ -0,0 +1,188 @@ +#!/usr/bin/env perl + +# use warnings; +# use strict; +use Carp; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use IPC::Cmd 0.36 qw( can_run run ); +use IO::Handle; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, $det_id, $camera); + +GetOptions('dbname=s' => \$dbname, + 'det_id=s' => \$det_id, + 'camera|c=s' => \$camera, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "USAGE: ipp_darkstats.pl --dbname (name) --det_id (id) --camera (name)", + -exitval => 3, + ) unless defined $dbname and defined $det_id and defined $camera; + +$ipprc->define_camera($camera); + +### Get list of dark imfile results + +# define the dettool command +my $command = "dettool -processedimfile -select_state stop"; # Command to run +$command .= " -det_id $det_id"; +$command .= " -dbname $dbname" if defined $dbname; + +# run the dettool command and catch the output +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", $error_code); +} + +# parse the output into a list +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $PS_EXIT_PROG_ERROR); +my $list = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $PS_EXIT_PROG_ERROR); + +my @bg_data; +my @bg_stdev_data; +my @bg_name; +my @bg_exptime; +my %components; + +print STDERR "extracted the data from the database\n"; + +# we now have a list of imfiles; we need to extract the background for each cell +# from the stats files for each imfile +foreach my $item (@$list) { + my $path_base = $item->{path_base}; + my $class_id = $item->{class_id}; + my $exp_time = $item->{exp_time}; + + my $rootName = $ipprc->file_resolve ($path_base); + my $statsName = "$rootName.$class_id.stats"; + + # print STDERR "rootName: $rootName : $exp_time\n"; + print STDERR "statsName: $statsName : $exp_time\n"; + + my $statsFile; + open $statsFile, $statsName; + my @contents = <$statsFile>; + close ($statsFile); + + # print STDERR "contents: @contents\n"; + + my $parser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $statsList = $parser->parse(join "", @contents) or &my_die("Unable to parse metadata for imfile stats", $PS_EXIT_SYS_ERROR); + + &parse_stats_table ($exp_time, $class_id, $statsList); +} + +print STDERR "parsed the stats from the data files\n"; + +for (my $i = 0; $i < @bg_data; $i++) { + $nameX = "$bg_name[$i].exp"; + $nameY = "$bg_name[$i].bg"; + push @{$nameX}, $bg_exptime[$i]; + push @{$nameY}, $bg_data[$i]; +} + +if (-e "output.dat") { unlink "output.dat"; } + +print STDERR "dumping stats\n"; +open (MANA, "|mana --norc"); +MANA->autoflush; + +foreach my $component (@components) { + $nameX = "$component.exp"; + $nameY = "$component.bg"; + + print MANA "delete X Y\n"; + + open (DATA, ">$component.dat"); + for (my $i = 0; $i < @{$nameX}; $i++) { + print DATA "${$nameX}[$i] ${$nameY}[$i]\n"; + } + close (DATA); + + print MANA "data $component.dat\n"; + print MANA "read X 1 Y 2\n"; + print MANA "fit X Y 2 -clip 3 3\n"; + print MANA "output output.dat\n"; + print MANA "echo $component METADATA\n"; + print MANA "echo \" NORDER_X S32 2 \"\n"; + print MANA "echo \" VAL_X00 F64 \$C0\"\n"; + print MANA "echo \" VAL_X01 F64 \$C1\"\n"; + print MANA "echo \" VAL_X02 F64 \$C2\"\n"; + print MANA "echo \" NELEMENTS S32 3 \"\n"; + print MANA "echo END\n"; + print MANA "echo\n"; + print MANA "output stdout\n"; + + print MANA "applyfit X Yf\n"; + print MANA "lim X Y\n"; + print MANA "clear\n"; + print MANA "box\n"; + print MANA "plot -x 2 -pt 2 -sz 1.0 -c black X Y\n"; + print MANA "plot -x 2 -pt 7 -sz 1.0 -c red X Yf\n"; + + print STDERR "hit return to continue\n"; + $answer = ; +} + +close (MANA); + +exit 0; + +sub parse_stats_table +{ + my ($exp_time, $tag, $md) = @_; + + # descend through the fpa + foreach my $entry (@$md) { + # print STDERR "name: $entry->{name}, class: $entry->{class}\n"; + # recurse on nested metadata + if ($entry->{class} eq 'metadata') { + my $newtag = $tag . "_" . $entry->{name}; + &parse_stats_table ($exp_time, $newtag, $entry->{value}); + } + + if ($entry->{name} =~ /^(SAMPLE|ROBUST|FITTED|CLIPPED)/) { + # It's a statistic of some sort + if ($entry->{name} =~ /_STDEV$/) { + push @bg_stdev_data, $entry->{value}; + } else { + push @bg_name, $tag; + push @bg_data, $entry->{value}; + push @bg_exptime, $exp_time; + # print STDERR "$tag $exp_time $entry->{value}\n"; + } + if (!$componentsHash{$tag}) { + push @components, $tag; + $componentsHash{$tag} = 1; + } + next; + } + } + return 1; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exit_code = shift; # Exit code to add + + carp($msg); + exit $exit_code; +} + +# - get the exp_time as well from dettool +# - build an array of bg & exptime for each cell +# - fit the trend (in mana? pslib functions?) +# - write the polynomial for each cell Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_datapath.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_datapath.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_datapath.pl (revision 22161) @@ -0,0 +1,25 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); + +use PS::IPP::Config; +my $ipprc = PS::IPP::Config->new(); + +my $touch; + +GetOptions( + 'touch' => \$touch, +); + +die "No filename specified.\n" if scalar @ARGV != 1; + +my $filename = shift @ARGV; + +my $resolved = $ipprc->file_resolve($filename, $touch); +print "$resolved\n" if ($resolved); + +1; + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_detrend_combine.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_detrend_combine.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_detrend_combine.pl (revision 22161) @@ -0,0 +1,220 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Data::Dumper; +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use PS::IPP::Metadata::Stats; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $RECIPE_PPSTATS = 'CHIPSTATS'; # Recipe to use with ppStats + +# Parse command-line arguments +my ($det_type, $filelevel, $inst, $telescope, $filter, + $det_id1, $iter1, $det_id2, $iter2, $operation, $mask, + $workdir, $dbname, $no_update); +GetOptions( + 'det_type=s' => \$det_type, # Detrend type for new detrend + 'filelevel=s' => \$filelevel, # File level for new detrend + 'inst=s' => \$inst, # Instrument for new detrend + 'telescope=s' => \$telescope, # Telescope for new detrend + 'filter=s' => \$filter, # Filter name for new detrend + 'det_id1=s' => \$det_id1, # Detrend id for detrend 1 + 'iteration1=s' => \$iter1, # Iteration for detrend 1 + 'det_id2=s' => \$det_id2, # Detrend id for detrend 2 + 'iteration2=s' => \$iter2, # Iteration for detrend 2 + 'operation=s' => \$operation, # Operation to perform on files + 'mask' => \$mask, # Operation is on a mask + 'workdir=s' => \$workdir, # Working directory for output files + 'dbname=s' => \$dbname, # Database name + 'no-update' => \$no_update, # Don't update the database + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options --det_type --filelevel --inst --telescope --det_id1 --iteration1 --det_id2 --iteration2 --workdir", + -exitval => 3, + ) + unless defined $det_type + and defined $filelevel + and defined $inst + and defined $telescope + and defined $det_id1 + and defined $iter1 + and defined $det_id2 + and defined $iter2 + and defined $operation + and defined $workdir; + +$ipprc->define_camera($inst); + +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; + +# Look for programs we need +my $missing_tools; +my $detselect = can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1); +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppArith = can_run('ppArith') or (warn "Can't find ppArith" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get the list of inputs +my $files1 = filelist($det_id1, $iter1); # Hash of input files for detrend 1 +my $files2 = filelist($det_id2, $iter2); # Hash of input files for detrend 2 +die("File lists for detrends have differing lengths") unless scalar keys %$files1 == scalar keys %$files2; + +my ($det_id, $iter); # Detrend identifier for the new detrend +unless ($no_update) { + my $command = "$dettool -register_detrend -det_type $det_type -filelevel $filelevel -workdir $workdir " . + "-inst $inst -telescope $telescope"; # Command to run + $command .= " -filter $filter" if defined $filter; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run dettool -register_detrend: $error_code"); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or die("Unable to parse metadata config doc\n"); + my $md = parse_md_list($metadata) or die("Unable to parse metadata list\n"); + + $det_id = $$md[0]->{det_id}; + $iter = $$md[0]->{iteration}; + + die("Unable to get det_id and iteration for new detrend.\n") unless defined $det_id and defined $iter; +} else { + $det_id = 'DUMMY_DET_ID'; + $iter = 'DUMMY_ITER'; +} + +my $outRoot = caturi($workdir, "$inst.$det_id.$iter"); # Output root name +my $filerule = (defined $mask ? "PPARITH.OUTPUT.MASK" : "PPARITH.OUTPUT.IMAGE"); # File rule for ppArith + +foreach my $class_id ( keys %$files1 ) { + my $md1 = $$files1{$class_id}; + my $md2 = $$files2{$class_id}; + die("Class_id=$class_id not defined for det_id=$det_id2") unless defined $md2; + + my $uri1 = $$md1[0]->{uri}; + my $uri2 = $$md2[0]->{uri}; + + die("Unable to find input file $uri1\n") unless $ipprc->file_exists($uri1); + die("Unable to find input file $uri2\n") unless $ipprc->file_exists($uri2); + + my $outName = $ipprc->filename($filerule, $outRoot, $class_id); + my $outStats = $outRoot . '.stats'; + + my $command = "$ppArith -file1 $uri1 -op \'$operation\' -file2 $uri2 $outRoot"; # Command to run + $command .= " -stats $outStats -recipe PPSTATS $RECIPE_PPSTATS"; + $command .= ' -mask' if defined $mask; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run ppArith: $error_code"); + } + + die("Unable to find ppArith product: $outName\n") unless $ipprc->file_exists($outName); + die("Unable to find ppArith product: $outStats\n") unless $ipprc->file_exists($outStats); + + # Get the statistics on the processed image + my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + { + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outStats) or die("Can't open stats file $outStats: $!"); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + my $metadata = $mdcParser->parse(join "", @contents) or die("Unable to parse metadata config doc"); + + unless ($stats->parse($metadata)) { + &my_die("Failure extracting metadata from the statistics output file.\n"); + } + } + + # Register the imfile + unless ($no_update) { + my $command = "$dettool -register_detrend_imfile -det_id $det_id "; # Command to run + $command .= " -class_id $class_id -uri $outName -path_base $outRoot"; + $command .= $stats->cmdflags(); + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run dettool -register_detrend_imfile: $error_code"); + } + } +} + + +### Pau. + + +# Get a list of files for the given detrend +sub filelist +{ + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + + my $command = "$detselect -select -det_id $det_id -iteration $iter"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run detselect: $error_code"); + } + + # Because of the length, need to split into individual metadatas --- it parses SO much quicker! + my %files; + + my $md = $mdcParser->parse( join( "", @$stdout_buf ) ); # Parsed metadata + my $list = parse_md_list( $md ); + + foreach my $item ( @$list ) { + my $class_id = $item->{class_id}; + die("Multiple definitions of class_id=$class_id found for det_id=$det_id, iteration=$iter\n") if + defined $files{$class_id}; + $files{$class_id} = parse_md_list($md); + } + + return \%files; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_filename.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_filename.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_filename.pl (revision 22161) @@ -0,0 +1,52 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# use these to check the apache environment +#print "$ENV{'PATH'}\n"; +#print "$ENV{'PERL5LIB'}\n"; + +use PS::IPP::Config; +my $ipprc = PS::IPP::Config->new(); + +my ($filerule, $class_id, $basename, $camera, $touch); + +GetOptions('filerule=s' => \$filerule, + 'class_id=s' => \$class_id, + 'basename=s' => \$basename, + 'camera|c=s' => \$camera, + 'touch' => \$touch, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --filerule --class_id --basename --camera", + -exitval => 3, +) unless defined $basename + and defined $filerule + and defined $class_id + and defined $camera; + +$touch = 0 unless (defined $touch); + +$ipprc->define_camera($camera); + +# print "$filerule\n"; +# print "$basename\n"; +# print "$camera\n"; +# print "$class_id\n"; + +my $filename = $ipprc->filename($filerule, $basename, $class_id); +# print "$filename\n"; +# print "touch: $touch\n"; + +my $realname = $ipprc->file_resolve( $filename, $touch ); +print "$realname\n"; + +1; + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_image_path.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_image_path.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_image_path.pl (revision 22161) @@ -0,0 +1,170 @@ +#!/usr/bin/env perl + +# ipp_image_path.pl print out the unix path name(s) for rawImage files for an exposure +# Note this program should really be called gpc1_image_path since the camera is +# hardcoded to gpc1 and the telescope is gpc1 +# + +use warnings; +use strict; + +# get images that are on ipp008 from alternative location until the +# dataase is updated +my $use_008_workaround = 1; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use DBI; +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($exp_name, $class_id, $from_registered, $dbname, $verbose); + +GetOptions( + 'exp_name|e=s' => \$exp_name, + 'class_id|c=s' => \$class_id, + 'registered|r' => \$from_registered, + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --exp_name", + -exitval => 3) unless + defined $exp_name; + +my $ota_num; +if ($class_id) { + # we accept class_id in 3 formats (all we care about is the number) + # xy23 + # ota23 + # 23 + $class_id = lc($class_id); + my $extra; + (undef, $ota_num, $extra) = $class_id =~ m/(\D*)(\d\d)(.*)/; + if ($extra || !$ota_num) { + print STDERR "$class_id is not a valid class_id\n"; + exit 1; + } +} + +# Look for commands we need +my $missing_tools; +my $neb_locate = can_run('neb-locate') + or (warn "can't find neb-locate" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $ipprc = PS::IPP::Config->new(); +my $dbh = getDBHandle(); + +# Get the list of imfiles +{ + # XXX: If there are multiple exposures with the same exposure name + # this query will return them all + my $query; + if ($from_registered) { + $query = "SELECT exp_id, class_id, uri FROM rawImfile" + . " WHERE exp_name = \'$exp_name\'" + . " AND camera = \'gpc1\' AND telescope = \'ps1\'"; + $query .= " AND class_id = \'xy$ota_num\'" if ($ota_num); + } else { + $query = "SELECT exp_id FROM newExp WHERE tmp_exp_name = ?" + . " AND tmp_camera = \'gpc1\' AND tmp_telescope = \'ps1\'"; + my $stmt = $dbh->prepare($query); + $stmt->execute($exp_name); + my $exp_ref = $stmt->fetchrow_hashref(); + if (!$exp_ref) { + print STDERR "exposure $exp_name not found\n"; + exit 1; + } + $stmt->finish(); + $query = "SELECT exp_id, tmp_class_id, uri FROM newImfile WHERE exp_id = $exp_ref->{exp_id}"; + $query .= " AND tmp_class_id = \'ota$ota_num\'" if ($ota_num); + } + + my $stmt = $dbh->prepare($query); + $stmt->execute(); + my $nfiles = 0; + while (my $ref = $stmt->fetchrow_hashref()) { + my $uri = $ref->{uri}; + my $path; + my ($scheme) = $uri =~/^(path|neb|file):/; + + if (!$scheme) { + $path = $uri; + } else { + if ($use_008_workaround && ($uri =~ /ipp008/)) { + $path = resolve_ipp008_file($uri); + } else { + $path = $ipprc->file_resolve($uri); + } + } + if ($path) { + # remove the leading "file://" if present + $path =~ s/^file\:\/\///; + } + $nfiles++; + print "$path\n"; + } + if (!$nfiles) { + my $cstr = defined($class_id) ? $class_id . " " : ""; + print STDERR "exposure $exp_name " . $cstr . "not found\n"; + exit 2; + } +} + +sub getDBHandle { + my $dbserver = metadataLookupStr($ipprc->{_siteConfig}, "DBSERVER"); + my $dbuser = metadataLookupStr($ipprc->{_siteConfig}, "DBUSER"); + my $dbpassword = metadataLookupStr($ipprc->{_siteConfig}, "DBPASSWORD"); + if (!$dbname) { + $dbname = metadataLookupStr($ipprc->{_siteConfig}, "DBNAME"); + } + + die "database configuration set up" unless defined($dbserver) and defined($dbuser) + and defined($dbpassword) and defined($dbname); + + my $dsn = "DBI:mysql:host=$dbserver;database=$dbname"; + + my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) + or die "Cannot connect to database.\n"; + + return $dbh; +} + +sub resolve_ipp008_file { + my %locations = ( "ota24" => "ipp006", + "ota25" => "ipp009", + "ota26" => "ipp011", + "ota27" => "ipp019", + "ota30" => "ipp020", + "ota31" => "ipp021" + ); + + my $uri = shift; + + # uri's look like neb://ipp008.0/gpc1/20080625/o4642g0400o/o4642g0400o.ota24.fits + + my ($left, $middle, $ota, $fits) = split '\.', $uri; + + my $node = $locations{$ota}; + + die "can't find node for $ota: $uri" if (!$node); + + my $path = $uri; + + $path =~ s%neb://ipp008.0%/data/${node}.0/recover08%; + + die "sorry backup image for $uri not found\n" if (!-e $path); + + return $path; +} Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_inject_expname.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_inject_expname.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_inject_expname.pl (revision 22161) @@ -0,0 +1,190 @@ +#!/usr/bin/env perl + +# this program injects a set of files the db, assuming the list of +# files all belong to a single exposure the user supplies a temporary +# telescope and camera name. these are used for informational +# purposes only until the registration step can determine the true +# telescope and camera name from the image headers. + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use PS::IPP::Config 1.01 qw( :standard ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +my $ipprc = PS::IPP::Config->new(); # this is used for PATH, NEB filename conversions + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ($camera, $telescope, $workdir, $reduction, $dvo_db, $tess_id, $end_stage, $label, $from_db, $dbname, $no_op, $verbose, $help); +GetOptions('camera|i=s' => \$camera, # user-supplied camera name + 'telescope|t=s' => \$telescope, # user-supplied telescope name + 'workdir|w=s' => \$workdir, # working directory for output files + 'reduction=s' => \$reduction, # user-supplied camera name + 'dvodb=s' => \$dvo_db, # target dvo database + 'tess_id=s' => \$tess_id, # tessalation for warping + 'end_stage=s' => \$end_stage, # stop processing at this step + 'label=s' => \$label, # set chip label + 'from=s' => \$from_db, # Source Database name + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # pretend but don't actually inject + 'help' => \$help # give help listing + ) or pod2usage( 2 ); + +pod2usage( -msg => "inject files based on query to another IPP db", + -exitval => 2) if + defined $help; + +pod2usage( -msg => "Usage: $0 [--from dbname] [--workdir path] [--reduction class] [--dvodb db] [--tess_id tess] [--end_stage stage] [--label label] [--dbname dbname] (exp_name)", + -exitval => 2 ) if + scalar @ARGV == 0; + +pod2usage( + -msg => "Required options: --exp_tag --cam_id --camera --outroot", + -exitval => 3, + ) unless + defined $from_db; + +my $exp_name = $ARGV[0]; + +my $pxinject = can_run('pxinject') or die "Can't find pxinject\n"; +my $regtool = can_run('regtool') or die "Can't find regtool\n"; + +# tess_id needs to be an absolute path, or it must start with file://, path://, or neb:// +if ($tess_id) { + $tess_id = &fixpath ($tess_id); +} + +# dvodb needs to be an absolute path, or it must start with file://, path://, or neb:// +if ($dvo_db) { + $dvo_db = &fixpath ($dvo_db); +} + +# workdir needs to be an absolute path, or it must start with file://, path://, or neb:// +if ($workdir) { + $workdir = &fixpath ($workdir); +} + +# if workdir is not defined, assign the current path +if (! $workdir) { + $workdir = File::Spec->rel2abs( "." ); +} + +if (! $telescope) { + $telescope = "UNKNOWN"; +} + +if (! $camera) { + $camera = "UNKNOWN"; +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# query the from_db for the desired imfiles: +# regtool -dbname $from_db -processedimfile -exp_name $exp_name + +# Get list of component files +my $files; # Array of component files +{ + my $command = "$regtool -dbname $from_db -processedimfile -exp_name $exp_name"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die "Unable to perform regtool for $exp_name: $error_code"; + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + die "Unable to parse metadata config doc $exp_name: $error_code"; + + # extract the metadata for the files into a hash list + $files = parse_md_list($metadata) or + die "Unable to parse metadata list: $exp_name: $error_code"; +} + +## inject the exposure + +# the telescope, instrument, and exp_name used here are temporary : register replaces them with the true values +my $command_exp = "$pxinject -newExp"; +$command_exp .= " -tmp_exp_name $exp_name"; +$command_exp .= " -tmp_inst $camera"; +$command_exp .= " -tmp_telescope $telescope"; +$command_exp .= " -workdir $workdir"; +$command_exp .= " -reduction $reduction" if defined $reduction; +$command_exp .= " -dvodb $dvo_db" if defined $dvo_db; +$command_exp .= " -tess_id $tess_id" if defined $tess_id; +$command_exp .= " -end_stage $end_stage" if defined $end_stage; +$command_exp .= " -label $label" if defined $label; +$command_exp .= " -dbname $dbname" if defined $dbname; + +my $exp_id = 0; +unless ($no_op) { + my ( $success_exp, $error_code_exp, $full_buf_exp, $stdout_buf_exp, $stderr_buf_exp ) = + run( command => $command_exp, verbose => 1 ); + die "Unable to inject $exp_name: $error_code_exp\n" if not $success_exp; + + my @line = split(/\s+/, $$stdout_buf_exp[0]); # The output line, containing the exposure tag + $exp_id = $line[2]; # The exposure tag +} else { + print "skipping command: $command_exp\n"; +} + +# now inject the imfiles one at a time +foreach my $file (@$files) { + + my $uri = $file->{uri}; + my $class_id = $file->{class_id}; + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_imfile = "$pxinject -newImfile"; + $command_imfile .= " -exp_id $exp_id"; + $command_imfile .= " -tmp_class_id $class_id"; + $command_imfile .= " -uri $uri"; + $command_imfile .= " -dbname $dbname" if defined ($dbname); + + unless ($no_op) { + my ( $success_imfile, $error_code_imfile, $full_buf_imfile, $stdout_buf_imfile, $stderr_buf_imfile ) = run( command => $command_imfile, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code_imfile\n" if not $success_imfile; + } else { + print "skipping command: $command_imfile\n"; + } +} + +# the class_id used here is temporary : register replaces it with the true class_id +my $command_update = "$pxinject -updatenewExp"; +$command_update .= " -exp_id $exp_id"; +$command_update .= " -state run"; +$command_update .= " -dbname $dbname" if defined ($dbname); + +unless ($no_op) { + my ( $success_update, $error_code_update, $full_buf_update, $stdout_buf_update, $stderr_buf_update ) = run( command => $command_update, verbose => 1 ); + die "Unable to update $exp_name: $error_code_update\n" if not $success_update; +} else { + print "skipping command: $command_update\n"; +} + +exit 0; + +sub fixpath { + my $path = shift; + my $valid = 0; + $valid |= ($path =~ m|^file://|); + $valid |= ($path =~ m|^path://|); + $valid |= ($path =~ m|^neb://|); + $valid |= ($path =~ m|^/|); + + if (!$valid) { + $path = File::Spec->rel2abs( $path ); + } + return $path; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_inject_fileset.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_inject_fileset.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_inject_fileset.pl (revision 22161) @@ -0,0 +1,177 @@ +#!/usr/bin/env perl + +# this program injects a set of files the db, assuming the list of +# files all belong to a single exposure the user supplies a temporary +# telescope and camera name. these are used for informational +# purposes only until the registration step can determine the true +# telescope and camera name from the image headers. + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use PS::IPP::Config; + +my $ipprc = PS::IPP::Config->new(); # this is used for PATH, NEB filename conversions + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ($camera, $telescope, $workdir, $reduction, $dvo_db, $tess_id, $end_stage, $label, $dbname, $no_op, $help); +GetOptions('camera|i=s' => \$camera, # user-supplied camera name + 'telescope|t=s' => \$telescope, # user-supplied telescope name + 'workdir|w=s' => \$workdir, # working directory for output files + 'reduction=s' => \$reduction, # user-supplied camera name + 'dvodb=s' => \$dvo_db, # target dvo database + 'tess_id=s' => \$tess_id, # tessalation for warping + 'end_stage=s' => \$end_stage, # stop processing at this step + 'label=s' => \$label, # set chip label + 'dbname|d=s' => \$dbname, # Database name + 'no-op' => \$no_op, # pretend but don't actually inject + 'help' => \$help # give help listing + ) or pod2usage( 2 ); + +pod2usage( -msg => "inject one or many files into the IPP pipeline database", + -exitval => 2) if + defined $help; + +pod2usage( -msg => "Usage: $0 --telescope (name) --camera (name) [--workdir path] [--reduction class] [--dvodb db] [--tess_id tess] [--end_stage stage] [--label label] [--dbname dbname] (files)", + -exitval => 2 ) if + scalar @ARGV == 0; + +# XXX why are these required? +pod2usage( -msg => "Required options: --telescope (name) --camera (name)", + -exitval => 3) unless @ARGV > 0; + +my $pxinject = can_run('pxinject') or die "Can't find pxinject\n"; + +# tess_id needs to be an absolute path, or it must start with file://, path://, or neb:// +if ($tess_id) { + $tess_id = &fixpath ($tess_id); +} + +# dvodb needs to be an absolute path, or it must start with file://, path://, or neb:// +if ($dvo_db) { + $dvo_db = &fixpath ($dvo_db); +} + +# workdir needs to be an absolute path, or it must start with file://, path://, or neb:// +if ($workdir) { + $workdir = &fixpath ($workdir); +} + +# if workdir is not defined, assign the current path +if (! $workdir) { + $workdir = File::Spec->rel2abs( "." ); +} + +if (! $telescope) { + $telescope = "UNKNOWN"; +} + +if (! $camera) { + $camera = "UNKNOWN"; +} + +# use the first file name as the exp_name (strip off .fits) +my $num = 0; +my $exp_name; +foreach my $file ( @ARGV ) { + # check for file existence + if (! -e $file) { die "file $file not found\n"; } + if (! $exp_name) { + # strip off the extension + my ( $vol, $path, $name ) = File::Spec->splitpath( $file ); + ( $exp_name ) = $name =~ /(.*)\.(fits|fit|fts)(|.gz)/; + print "exp_name : $exp_name.\n"; + } + $num ++; +} + +print "$num files in fileset.\n"; + +## inject the exposure + +# the telescope, instrument, and exp_name used here are temporary : register replaces them with the true values +my $command_exp = "$pxinject -newExp"; +$command_exp .= " -tmp_exp_name $exp_name"; +$command_exp .= " -tmp_inst $camera"; +$command_exp .= " -tmp_telescope $telescope"; +$command_exp .= " -workdir $workdir"; +$command_exp .= " -reduction $reduction" if defined $reduction; +$command_exp .= " -dvodb $dvo_db" if defined $dvo_db; +$command_exp .= " -tess_id $tess_id" if defined $tess_id; +$command_exp .= " -end_stage $end_stage" if defined $end_stage; +$command_exp .= " -label $label" if defined $label; +$command_exp .= " -dbname $dbname" if defined $dbname; + +my $exp_id = 0; +unless ($no_op) { + my ( $success_exp, $error_code_exp, $full_buf_exp, $stdout_buf_exp, $stderr_buf_exp ) = + run( command => $command_exp, verbose => 1 ); + die "Unable to inject $exp_name: $error_code_exp\n" if not $success_exp; + + my @line = split(/\s+/, $$stdout_buf_exp[0]); # The output line, containing the exposure tag + $exp_id = $line[2]; # The exposure tag +} else { + print "skipping command: $command_exp\n"; +} + +# now inject the imfiles one at a time +for (my $i = 0; $i < @ARGV; $i++) { + + my $file = $ARGV[$i]; + + my $absfile = File::Spec->rel2abs( $file ); + my $relfile = $ipprc->convert_filename_relative( $absfile ); + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_imfile = "$pxinject -newImfile"; + $command_imfile .= " -exp_id $exp_id"; + $command_imfile .= " -tmp_class_id file.$i"; + $command_imfile .= " -uri $relfile"; + $command_imfile .= " -dbname $dbname" if defined ($dbname); + + unless ($no_op) { + my ( $success_imfile, $error_code_imfile, $full_buf_imfile, $stdout_buf_imfile, $stderr_buf_imfile ) = run( command => $command_imfile, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code_imfile\n" if not $success_imfile; + } else { + print "skipping command: $command_imfile\n"; + } +} + +# the class_id used here is temporary : register replaces it with the true class_id +my $command_update = "$pxinject -updatenewExp"; +$command_update .= " -exp_id $exp_id"; +$command_update .= " -state run"; +$command_update .= " -dbname $dbname" if defined ($dbname); + +unless ($no_op) { + my ( $success_update, $error_code_update, $full_buf_update, $stdout_buf_update, $stderr_buf_update ) = run( command => $command_update, verbose => 1 ); + die "Unable to update $exp_name: $error_code_update\n" if not $success_update; +} else { + print "skipping command: $command_update\n"; +} + +exit 0; + +sub fixpath { + my $path = shift; + my $valid = 0; + $valid |= ($path =~ m|^file://|); + $valid |= ($path =~ m|^path://|); + $valid |= ($path =~ m|^neb://|); + $valid |= ($path =~ m|^/|); + + if (!$valid) { + $path = File::Spec->rel2abs( $path ); + } + return $path; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_maskscript.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_maskscript.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_maskscript.pl (revision 22161) @@ -0,0 +1,190 @@ +#!/usr/bin/env perl + +# use warnings; +# use strict; +use Carp; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use IPC::Cmd 0.36 qw( can_run run ); +use IO::Handle; + +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, $det_id, $camera); + +GetOptions('dbname=s' => \$dbname, + 'det_id=s' => \$det_id, + 'camera|c=s' => \$camera, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "USAGE: ipp_maskscript.pl --dbname (name) --det_id (id) --iter (iteration) --camera (name)", + -exitval => 3, + ) unless defined $dbname and defined $det_id and defined $camera; + +# I could determine the camera from a query for the detrun +$ipprc->define_camera($camera); + +### Get list of dark imfile results + +# define the dettool command +my $command = "dettool -processedimfile -select_state stop"; # Command to run +$command .= " -det_id $det_id"; +$command .= " -dbname $dbname" if defined $dbname; + +# run the dettool command and catch the output +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", $error_code); +} + +# parse the output into a list +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $PS_EXIT_PROG_ERROR); +my $list = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $PS_EXIT_PROG_ERROR); + +my @bg_data; +my @bg_stdev_data; +my @bg_name; +my @bg_exptime; +my %components; + +print STDERR "extracted the data from the database\n"; + +# we now have a list of imfiles; we need to extract the background for each cell +# from the stats files for each imfile +foreach my $item (@$list) { + my $path_base = $item->{path_base}; + my $class_id = $item->{class_id}; + my $exp_time = $item->{exp_time}; + + my $rootName = $ipprc->file_resolve ($path_base); + my $statsName = "$rootName.$class_id.stats"; + + # print STDERR "rootName: $rootName : $exp_time\n"; + print STDERR "statsName: $statsName : $exp_time\n"; + + my $statsFile; + open $statsFile, $statsName; + my @contents = <$statsFile>; + close ($statsFile); + + # print STDERR "contents: @contents\n"; + + my $parser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $statsList = $parser->parse(join "", @contents) or &my_die("Unable to parse metadata for imfile stats", $PS_EXIT_SYS_ERROR); + + &parse_stats_table ($exp_time, $class_id, $statsList); +} + +print STDERR "parsed the stats from the data files\n"; + +for (my $i = 0; $i < @bg_data; $i++) { + $nameX = "$bg_name[$i].exp"; + $nameY = "$bg_name[$i].bg"; + push @{$nameX}, $bg_exptime[$i]; + push @{$nameY}, $bg_data[$i]; +} + +if (-e "output.dat") { unlink "output.dat"; } + +print STDERR "dumping stats\n"; +open (MANA, "|mana --norc"); +MANA->autoflush; + +foreach my $component (@components) { + $nameX = "$component.exp"; + $nameY = "$component.bg"; + + print MANA "delete X Y\n"; + + open (DATA, ">$component.dat"); + for (my $i = 0; $i < @{$nameX}; $i++) { + print DATA "${$nameX}[$i] ${$nameY}[$i]\n"; + } + close (DATA); + + print MANA "data $component.dat\n"; + print MANA "read X 1 Y 2\n"; + print MANA "fit X Y 2 -clip 3 3\n"; + print MANA "output output.dat\n"; + print MANA "echo $component METADATA\n"; + print MANA "echo \" NORDER_X S32 2 \"\n"; + print MANA "echo \" VAL_X00 F64 \$C0\"\n"; + print MANA "echo \" VAL_X01 F64 \$C1\"\n"; + print MANA "echo \" VAL_X02 F64 \$C2\"\n"; + print MANA "echo \" NELEMENTS S32 3 \"\n"; + print MANA "echo END\n"; + print MANA "echo\n"; + print MANA "output stdout\n"; + + print MANA "applyfit X Yf\n"; + print MANA "lim X Y\n"; + print MANA "clear\n"; + print MANA "box\n"; + print MANA "plot -x 2 -pt 2 -sz 1.0 -c black X Y\n"; + print MANA "plot -x 2 -pt 7 -sz 1.0 -c red X Yf\n"; + + print STDERR "hit return to continue\n"; + $answer = ; +} + +close (MANA); + +exit 0; + +sub parse_stats_table +{ + my ($exp_time, $tag, $md) = @_; + + # descend through the fpa + foreach my $entry (@$md) { + # print STDERR "name: $entry->{name}, class: $entry->{class}\n"; + # recurse on nested metadata + if ($entry->{class} eq 'metadata') { + my $newtag = $tag . "_" . $entry->{name}; + &parse_stats_table ($exp_time, $newtag, $entry->{value}); + } + + if ($entry->{name} =~ /^(SAMPLE|ROBUST|FITTED|CLIPPED)/) { + # It's a statistic of some sort + if ($entry->{name} =~ /_STDEV$/) { + push @bg_stdev_data, $entry->{value}; + } else { + push @bg_name, $tag; + push @bg_data, $entry->{value}; + push @bg_exptime, $exp_time; + # print STDERR "$tag $exp_time $entry->{value}\n"; + } + if (!$componentsHash{$tag}) { + push @components, $tag; + $componentsHash{$tag} = 1; + } + next; + } + } + return 1; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exit_code = shift; # Exit code to add + + carp($msg); + exit $exit_code; +} + +# - get the exp_time as well from dettool +# - build an array of bg & exptime for each cell +# - fit the trend (in mana? pslib functions?) +# - write the polynomial for each cell Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_mops_translate.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_mops_translate.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_mops_translate.pl (revision 22161) @@ -0,0 +1,263 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use Astro::FITS::CFITSIO qw( :constants ); +Astro::FITS::CFITSIO::PerlyUnpacking(1); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use Math::Trig; +use Data::Dumper; + +use constant EXTNAME => 'MOPS_TRANSIENT_DETECTIONS'; # Extension name for output table +use constant POSITION_ERROR => 0.2 / 3600; # Error in positions, degrees +use constant ZERO_POINT => 25; # Magnitude zero point +use constant MAG_ERROR => 0.1; # Error in magnitudes +use constant OBSERVATORY_CODE => 566; # IAU Observatory Code +use constant FAKE_LIMITING_MAG => 23.0; # Fake limiting magnitude to report +use constant FAKE_DETECTION_EFFICIENCY => 0.0; # Fake detection efficiency to report + +my ( $input, # Name of input file with IPP photometry + $extname, # Name of extension containing photometry + $skycell, # Skycell file with WCS + $output, # Name of output file + $save_temps, # Save temporary files? + ); + +GetOptions( + 'input=s' => \$input, + 'extname=s' => \$extname, + 'skycell=s' => \$skycell, + 'output=s' => \$output, + 'save-temps' => \$save_temps, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --input --extname --skycell --output", + -exitval => 3) + unless defined $input + and defined $extname + and defined $skycell + and defined $output; + +# Specification of columns to write +my $columns = [ { name => 'RA_DEG', type => 'D' }, # Right ascension + { name => 'DEC_DEG', type => 'D' }, # Declination + { name => 'RA_SIG', type => 'D' }, # Error in right ascension + { name => 'DEC_SIG', type => 'D' }, # Error in declination + { name => 'FLUX', type => 'D' }, # Flux + { name => 'FLUX_SIG', type => 'D' }, # Error in flux + { name => 'ANG', type => 'D' }, # Angle + { name => 'ANG_SIG', type => 'D' }, # Error in angle + { name => 'LEN', type => 'D' }, # Length + { name => 'LEN_SIG', type => 'D' }, # Error in length + ]; + +# Header translation table +my $headers = { + 'FPA.RA' => { name => 'RA', type => TSTRING, comment => 'Right ascension of boresight'}, + 'FPA.DEC' => { name => 'DEC', type => TSTRING, comment => 'Declination of boresight' }, + 'FPA.OBS' => { name => 'FPA_ID', type => TSTRING, comment => 'Exposure identifier' }, + 'FPA.FILTER' => { name => 'FILTER', type => TSTRING, comment => 'Filter name' }, + 'EXPTIME' => { name => 'EXPTIME', type => TDOUBLE, comment => 'Exposure time' }, + 'FPA.POSANGLE' => { name => 'ROTANGLE', type => TDOUBLE, comment => 'Position angle' }, + 'FPA.ALT' => { name => 'TEL_ALT', type => TDOUBLE, comment => 'Telescope altitude' }, + 'FPA.AZ' => { name => 'TEL_AZ', type => TDOUBLE, comment => 'Telescope azimuth' }, + }; + + +# Read the skycell header for the WCS +my ($wcsheader, $status) = Astro::FITS::CFITSIO::fits_read_header( $skycell ); + +# Get the useful header keywords +my $naxis1 = $$wcsheader{'NAXIS1'} or die("Can't find NAXIS1"); +my $naxis2 = $$wcsheader{'NAXIS2'} or die("Can't find NAXIS2"); +my $ctype1 = $$wcsheader{'CTYPE1'} or die("Can't find CTYPE1"); +my $ctype2 = $$wcsheader{'CTYPE2'} or die("Can't find CTYPE2"); +my $cdelt1 = $$wcsheader{'CDELT1'} or die("Can't find CDELT1"); +my $cdelt2 = $$wcsheader{'CDELT2'} or die("Can't find CDELT2"); +my $crval1 = deg2rad($$wcsheader{'CRVAL1'}) or die("Can't find CRVAL1"); +my $crval2 = deg2rad($$wcsheader{'CRVAL2'}) or die("Can't find CRVAL2"); +my $crpix1 = $$wcsheader{'CRPIX1'} or die("Can't find CRPIX1"); +my $crpix2 = $$wcsheader{'CRPIX2'} or die("Can't find CRPIX2"); +my $pc11 = $$wcsheader{'PC001001'} or die("Can't find PC001001"); +my $pc12 = $$wcsheader{'PC001002'} or die("Can't find PC001002"); +my $pc21 = $$wcsheader{'PC002001'} or die("Can't find PC002001"); +my $pc22 = $$wcsheader{'PC002002'} or die("Can't find PC002002"); +my $crota1 = $$wcsheader{'CROTA1'}; +my $crota2 = $$wcsheader{'CROTA2'}; + +die("Unexpected projection: $ctype1 and $ctype2.") unless $ctype1 =~ /^\'RA---TAN\s*\'$/ and + $ctype2 =~ /^\'DEC--TAN\s*\'$/; +die("Can't determine size of skycell ($naxis1,$naxis2)") unless $naxis1 > 0 and $naxis2 > 0; +die("Can't determine scale of skycell ($cdelt1,$cdelt2)") if not defined $cdelt1 or $cdelt1 == 0 or + not defined $cdelt2 or $cdelt2 == 0; +die("We don't know how to handle rotations ($crota1,$crota2)") if defined $crota1 or defined $crota2; + + +# Read the input table +my $inFits = Astro::FITS::CFITSIO::open_file( $input, READONLY, $status ); # FITS file handle +my $inHeader = $inFits->read_header(); # Header for input +check_fitsio($status); +$inFits->movnam_hdu(BINARY_TBL, $extname, 0, $status) and check_fitsio($status); +my $numRows; # Number of rows in table +$inFits->get_num_rows($numRows, $status) and check_fitsio($status); + +my ($xCol, $yCol, $magCol); # Column numbers for x and y +$inFits->get_colnum(0, 'X_PSF', $xCol, $status) and check_fitsio($status); +$inFits->get_colnum(0, 'Y_PSF', $yCol, $status) and check_fitsio($status); +$inFits->get_colnum(0, 'PSF_INST_MAG', $magCol, $status) and check_fitsio($status); + +my ($xType, $yType, $magType); # Column types +$inFits->get_coltype($xCol, $xType, undef, undef, $status) and check_fitsio($status); +$inFits->get_coltype($yCol, $yType, undef, undef, $status) and check_fitsio($status); +$inFits->get_coltype($magCol, $magType, undef, undef, $status) and check_fitsio($status); + +my ($x, $y, $mag); # Sources arrays +$inFits->read_col($xType, $xCol, 1, 1, $numRows, 0, $x, undef, $status) and check_fitsio($status); +$inFits->read_col($yType, $yCol, 1, 1, $numRows, 0, $y, undef, $status) and check_fitsio($status); +$inFits->read_col($magType, $magCol, 1, 1, $numRows, 0, $mag, undef, $status) and check_fitsio($status); + +$inFits->close_file( $status ); + +# Parse the list of columns +my @colNames; # Names of columns +my @colTypes; # Types of columns +my %colData; # Data for each column +foreach my $colSpec ( @$columns) { + push @colNames, $colSpec->{name}; + push @colTypes, $colSpec->{type}; + $colData{$colSpec->{name}} = []; +} + + +# Convert the input data into the output formats +for (my $i = 0; $i < $numRows; $i++) { + my ($ra, $dec) = pixels_to_sky($$x[$i], $$y[$i]); # Sky coordinates + + push @{$colData{'RA_DEG'}}, $ra; + push @{$colData{'DEC_DEG'}}, $dec; + push @{$colData{'RA_SIG'}}, POSITION_ERROR; + push @{$colData{'DEC_SIG'}}, POSITION_ERROR; + push @{$colData{'FLUX'}}, $$mag[$i] + ZERO_POINT; + push @{$colData{'FLUX_SIG'}}, MAG_ERROR; + push @{$colData{'ANG'}}, 0.0; + push @{$colData{'ANG_SIG'}}, 0.0; + push @{$colData{'LEN'}}, 0.0; + push @{$colData{'LEN_SIG'}}, 0.0; +} + + +# Write the output +unlink "$output" if -e "$output"; +my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status ); # Output file handle +check_fitsio( $status ); + +# Write the table +$outFits->create_tbl( BINARY_TBL(), $numRows, scalar @colNames, \@colNames, \@colTypes, undef, EXTNAME, + $status ); +check_fitsio( $status ); + +# Write the header keywords +foreach my $keyword ( keys %$headers ) { + my $value = $inHeader->{$keyword}; # Header keyword value + unless (defined $value) { + print "Can't find header keyword $keyword\n"; + next; + } + $value =~ s/\'//g; + my $name = $headers->{$keyword}->{name}; # New name + my $type = $headers->{$keyword}->{type}; # Type + my $comment = $headers->{$keyword}->{comment}; # Comment + $outFits->write_key( $type, $name, $value, $comment, $status ); + check_fitsio( $status ); +} + +# Adjust the time from start to mid-point +{ + my $mjd = $inHeader->{'MJD-OBS'} or die "Can't find MJD.\n"; # Modified Julian Date + my $exptime = $inHeader->{'EXPTIME'} or die "Can't find EXPTIME.\n"; # Exposure time, seconds + $mjd += $exptime / 2.0; # start --> mid-point + $outFiles->write_key( TDOUBLE, 'MJD-OBS', $mjd, 'Time of exposure mid-point', $status); + check_fitsio( $status ); +} + +# Stellar PSF +my $fwhm = 0.5 * 3600 * ($inHeader->{'FWHM_X'} * $cdelt1 + $inHeader->{'FWHM_Y'} * $cdelt2); # FWHM for star +$outFits->write_key( TSTRING, 'STARPSF', $fwhm, 'Stellar PSF (arcsec)', $status ); +check_fitsio( $status ); + +# Observatory code +$outFits->write_key( TINT, 'OBSCODE', OBSERVATORY_CODE, 'IAU observatory code', $status ); + +# Limiting magnitude +$outFits->write_key( TINT, 'LIMITMAG', FAKE_LIMITING_MAG, 'Limiting magnitude (FAKE)', $status ); + +# Detection efficiency +$outFits->write_key( TINT, 'DE1', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); +$outFits->write_key( TINT, 'DE2', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); +$outFits->write_key( TINT, 'DE3', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); +$outFits->write_key( TINT, 'DE4', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); +$outFits->write_key( TINT, 'DE5', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); +$outFits->write_key( TINT, 'DE6', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); +$outFits->write_key( TINT, 'DE7', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); +$outFits->write_key( TINT, 'DE8', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); +$outFits->write_key( TINT, 'DE9', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); +$outFits->write_key( TINT, 'DE10', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); + +# Write the data +for (my $i = 0; $i < scalar @colNames; $i++) { + my $colName = $colNames[$i];# Column name + $outFits->write_col( TDOUBLE, $i + 1, 1, 1, $numRows, $colData{$colName}, $status ); + check_fitsio( $status ); +} + +$outFits->close_file( $status ); + +### Pau. + + + +# From Astro::FITS::CFITSIO demo +sub check_fitsio +{ + my $status = shift; # Status of FITSIO calls + + if ($status != 0) { + my $msg; # Message to output + Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg ); + die "CFITSIO error: $msg\n"; + } +} + +# Convert pixel coordinates to sky coordinates +sub pixels_to_sky +{ + my ($x, $y) = @_; # Coordinates + + # Pixel coordinate relative to reference + $x -= $crpix1; + $y -= $crpix2; + + # Coordinates on tangent plane + my $xi = $pc11 * ($x) + $pc12 * ($y); + my $eta = $pc21 * ($x) + $pc22 * ($y); + $xi *= $cdelt1; + $eta *= $cdelt2; + + # Coordinates on rotated celestial sphere + my $phi = atan2($eta,$xi) + pi/2; + my $theta = atan(180 / pi / sqrt($xi**2 + $eta**2)); + + # Coordinates on celestial sphere + my $ra = $crval1 + atan2(cos($theta) * sin($phi), + sin($theta) * cos($crval2) + cos($theta) * sin($crval2) * cos($phi)); + my $dec = asin(sin($theta) * sin($crval2) - cos($theta) * cos($crval2) * cos($phi)); + + return (rad2deg($ra), rad2deg($dec)); +} + + +__END__ + Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_camera.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_camera.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_camera.pl (revision 22161) @@ -0,0 +1,85 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config qw( caturi ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dbname, # Database name to use + $workdir_default, # Default working directory + $verbose, # Verbose operations? + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname=s' => \$dbname, + 'workdir=s' => \$workdir_default, + 'verbose' => \$verbose, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +$workdir_default = `pwd` unless defined $workdir_default; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser +my $ipprc = PS::IPP::Config->new; # IPP Configuration + +# Look for programs we need +my $missing_tools; +my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); +my $camera_exp = can_run('camera_exp.pl') or (warn "Can't find camera_exp.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Camera exposure processing +my $list; +{ + my $command = "$camtool -pendingexp -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get camera exposure list: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from camtool.\n"; +} + +foreach my $item (@$list) { + my $cam_id = $item->{cam_id}; + my $exp_tag = $item->{exp_tag}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_default unless (defined $workdir or $workdir ne "NULL"); + my $outroot = caturi( $workdir, $exp_tag, "$exp_tag.cm.$cam_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$camera_exp --cam_id $cam_id --exp_tag $exp_tag --camera $camera --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do camera processing on $cam_id: $error_code\n" if not $success; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +__END__ + + Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_chip.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_chip.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_chip.pl (revision 22161) @@ -0,0 +1,105 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config qw( caturi ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dbname, # Database name to use + $workdir_default, # Default working directory + $verbose, # Verbose operations? + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname=s' => \$dbname, + 'workdir=s' => \$workdir_default, + 'verbose' => \$verbose, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( -msg => "Required options: --dbname --workdir", + -exitval => 3, + ) unless + defined $dbname; + +$workdir_default = `pwd` unless defined $workdir_default; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser +my $ipprc = PS::IPP::Config->new; # IPP Configuration + +# Look for programs we need +my $missing_tools; +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $chip = can_run('chip_imfile.pl') or (warn "Can't find chip_imfile.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Imfile processing +my @whole; # The whole list for processing +{ + my $command = "$chiptool -pendingimfile -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get phase 2 imfile list: $error_code\n" if not $success; + @whole = split /\n/, join( '', @$stdout_buf ); +} + +my @single = (); + +while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from chiptool.\n"; + + foreach my $item (@$list) { + my $chip_id = $item->{chip_id}; + my $exp_id = $item->{exp_id}; + my $exp_tag = $item->{exp_tag}; + my $camera = $item->{camera}; + my $class_id = $item->{class_id}; + my $uri = $item->{uri}; + my $reduction = $item->{reduction}; + my $workdir = $item->{workdir}; + $workdir = $workdir_default unless (defined $workdir or $workdir ne "NULL"); + + my $outroot = caturi( $workdir, $exp_tag, "$exp_tag.ch.$chip_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$chip --chip_id $chip_id --exp_id $exp_id --class_id $class_id --uri $uri --dbname $dbname --camera $camera --outroot $outroot"; + $command .= " --reduction $reduction" if defined $reduction; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do phase 2 processing on $chip_id $class_id: $error_code\n" if not $success; + } + + @single = (); + + } +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +__END__ + + Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_detrend.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_detrend.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_detrend.pl (revision 22161) @@ -0,0 +1,377 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use Getopt::Long; +use Pod::Usage qw( pod2usage ); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config qw( caturi ); +use Data::Dumper; + +my ($dbname, # Database name to use + $workdir_global, # Global working directory + $verbose, # Produce verbose output? + ); +GetOptions( + 'dbname=s' => \$dbname, + 'workdir=s' => \$workdir_global, + 'verbose' => \$verbose, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $detrend_process_imfile = can_run('detrend_process_imfile.pl') or (warn "Can't find detrend_process_imfile.pl" and $missing_tools = 1); +my $detrend_process_exp = can_run('detrend_process_exp.pl') or (warn "Can't find detrend_process_exp.pl" and $missing_tools = 1); +my $detrend_stack = can_run('detrend_stack.pl') or (warn "Can't find detrend_stack.pl" and $missing_tools = 1); +my $detrend_norm_calc = can_run('detrend_norm_calc.pl') or (warn "Can't find detrend_norm_calc.pl" and $missing_tools = 1); +my $detrend_norm_apply = can_run('detrend_norm_apply.pl') or (warn "Can't find detrend_norm_apply.pl" and $missing_tools = 1); +my $detrend_norm_exp = can_run('detrend_norm_exp.pl') or (warn "Can't find detrend_norm_exp.pl" and $missing_tools = 1); +my $detrend_resid_imfile = can_run('detrend_resid_imfile.pl') or (warn "Can't find detrend_resid_imfile.pl" and $missing_tools = 1); +my $detrend_resid_exp = can_run('detrend_resid_exp.pl') or (warn "Can't find detrend_resid_exp.pl" and $missing_tools = 1); +my $detrend_reject_exp = can_run('detrend_reject_exp.pl') or (warn "Can't find detrend_reject_exp.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Process raw imfiles +{ + my $command = "$dettool -toprocessedimfile -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get detrend raw list: $error_code\n" if not $success; + + my @whole = split /\n/, join( '', @$stdout_buf ); + my @single = (); + + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $det_type = $item->{det_type}; + my $exp_tag = $item->{exp_tag}; + my $exp_id = $item->{exp_id}; + my $class_id = $item->{class_id}; + my $uri = $item->{uri}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + my $reduction = $item->{reduction}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", $exp_tag, "$exp_tag.detproc.$det_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_process_imfile --det_id $det_id --exp_tag $exp_tag --exp_id $exp_id --class_id $class_id --det_type $det_type --input_uri $uri --camera $camera --dbname $dbname --outroot $outroot"; + $command .= " --reduction $reduction" if defined $reduction; + $command .= " --verbose" if defined $verbose; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do raw imfile processing on $exp_tag $class_id: $error_code\n" if not $success; + } + + @single = (); + + } + } +} + +# Process raw exposures +{ + my $command = "$dettool -toprocessedexp -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get detrend raw list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $exp_tag = $item->{exp_tag}; + my $exp_id = $item->{exp_id}; + my $camera = $item->{camera}; + my $det_id = $item->{det_id}; + my $det_type = $item->{det_type}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", $exp_tag, "$exp_tag.detproc.$det_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_process_exp --det_id $det_id --det_type $det_type --exp_tag $exp_tag --exp_id $exp_id --camera $camera --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do raw exposure processing on $det_id $exp_tag: $error_code\n" if not $success; + } +} + +# Stack +{ + my $command = "$dettool -tostacked -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get stack list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $class_id = $item->{class_id}; + my $det_type = $item->{det_type}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + my $reduction = $item->{reduction}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", "$camera.$det_type.$det_id.$iteration" ); + $ipprc->outroot_prepare( $outroot ); + + + my $command = "$detrend_stack --det_id $det_id --iteration $iteration --class_id $class_id --det_type $det_type --camera $camera --dbname $dbname --outroot $outroot"; + $command .= " --reduction $reduction" if defined $reduction; + $command .= " --verbose" if defined $verbose; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to stack detrend $det_id $iteration $class_id: $error_code\n" if not $success; + } +} + +# Calculate normalisation +{ + my $command = "$dettool -tonormalizedstat -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get normalise calculation list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", "$camera.$det_type.normstat.$det_id.$iteration" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_norm_calc --det_id $det_id --iteration $iteration --det_type $det_type --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to calculate normalisation for $det_id $iteration: $error_code\n" if not $success; + } +} + +# Apply normalisation +{ + my $command = "$dettool -tonormalize -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get normalisation list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $class_id = $item->{class_id}; + my $value = $item->{norm}; + my $uri = $item->{uri}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", "$camera.$det_type.norm.$det_id.$iteration" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_norm_apply --det_id $det_id --iteration $iteration --class_id $class_id --value $value --input_uri $uri --camera $camera --det_type $det_type --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to apply normalisation for $det_id $iteration $class_id: $error_code\n" if not $success; + } +} + +# Examine normalised exposure +{ + my $command = "$dettool -tonormalizedexp -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get normalised exposures list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $class_id = $item->{class_id}; + my $value = $item->{norm}; + my $uri = $item->{uri}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", "$camera.$det_type.normexp.$det_id.$iteration" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_norm_exp --det_id $det_id --iteration $iteration --camera $camera --det_type $det_type --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to examine normalised exposure for $det_id $iteration: $error_code\n" if not $success; + } +} + +# Get residuals +{ + my $command = "$dettool -toresidimfile -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get residual processing list: $error_code\n" if not $success; + + my @whole = split /\n/, join( '', @$stdout_buf ); + my @single = (); + + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $exp_tag = $item->{exp_tag}; + my $exp_id = $item->{exp_id}; + my $camera = $item->{camera}; + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $class_id = $item->{class_id}; + my $det_type = $item->{det_type}; + my $detrend = $item->{det_uri}; + my $uri = $item->{uri}; + my $mode = $item->{mode}; + my $workdir = $item->{workdir}; + my $reduction = $item->{reduction}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", $exp_tag, "$exp_tag.detresid.$det_id.$iteration" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_resid_imfile --det_id $det_id --iteration $iteration --exp_tag $exp_tag --exp_id $exp_id --class_id $class_id --det_type $det_type --camera $camera --input_uri $uri --mode $mode --dbname $dbname --outroot $outroot"; + $command .= " --reduction $reduction" if defined $reduction; + $command .= " --detrend $detrend" if defined $detrend; + $command .= " --verbose" if defined $verbose; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do residual processing on $exp_tag $class_id: $error_code\n" if not $success; + } + + @single = (); + + } + } +} + +# Reject based on imfiles +{ + my $command = "$dettool -toresidexp -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get residual imfile list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $exp_tag = $item->{exp_tag}; + my $exp_id = $item->{exp_id}; + my $camera = $item->{camera}; + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", $exp_tag, "$exp_tag.detresid.$det_id.$iteration" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_resid_exp --det_id $det_id --iteration $iteration --exp_tag $exp_tag --exp_id $exp_id --det_type $det_type --camera $camera --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do imfile rejection on $exp_tag $det_id $iteration: $error_code\n" if not $success; + } +} + +# Reject based on exposures +{ + my $command = "$dettool -todetrunsummary -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get residual exposure list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $camera = $item->{camera}; + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", "$camera.$det_type.$det_id.$iteration.detreject" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_reject_exp --det_id $det_id --iteration $iteration --det_type $det_type --camera $camera --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do exposure rejection on $det_id $iteration: $error_code\n" if not $success; + } +} + + +__END__ + + Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_diff.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_diff.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_diff.pl (revision 22161) @@ -0,0 +1,96 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Basename; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, # Database name to use + $verbose, # Verbose operations? + $workdir_global, # Global working directory + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname=s' => \$dbname, + 'verbose' => \$verbose, + 'workdir' => \$workdir_global, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $difftool = can_run('difftool') or + (warn "Can't find difftool" and $missing_tools = 1); +my $diff_skycell = can_run('diff_skycell.pl') or + (warn "Can't find diff_skycell.pl" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + + +# Image differencing +{ + my $list; + my $command = "$difftool -todiffskyfile -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get list of diffs: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from difftool.\n"; + + foreach my $item (@$list) { + my $diff_id = $item->{diff_id}; + my $skycell_id = $item->{skycell_id}; + my $tess_id = basename($item->{tess_id}); + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, $tess_id, $skycell_id, "$tess_id.$skycell_id.dif.$diff_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$diff_skycell --diff_id $diff_id --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do difference for $diff_id: $error_code\n" if not $success; + } +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_inject.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_inject.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_inject.pl (revision 22161) @@ -0,0 +1,142 @@ +#!/usr/bin/env perl + +# this program injects a list of single-file exposures into the db, +# taking the filename (without .fits) as the exp_tag. the user +# supplies a temporary telescope and camera name. these are used +# for informational purposes only until the registration step can +# determine the true telescope and camera name from the image +# headers. + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use PS::IPP::Config; + +my $ipprc = PS::IPP::Config->new(); # this is used for PATH, NEB filename conversions + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ($camera, $telescope, $workdir, $reduction, $dvo_db, $tess_id, $end_stage, $label, $dbname, $no_op, $help); +GetOptions('camera|i=s' => \$camera, # user-supplied camera name + 'telescope|t=s' => \$telescope, # user-supplied telescope name + 'workdir|w=s' => \$workdir, # working directory for output files + 'reduction=s' => \$reduction, # user-supplied camera name + 'dvodb=s' => \$dvo_db, # target dvo database + 'tess_id=s' => \$tess_id, # tessalation for warping + 'end_stage=s' => \$end_stage, # stop processing at this step + 'label=s' => \$label, # set chip label + 'dbname|d=s' => \$dbname, # Database name + 'no-op' => \$no_op, # pretend but don't actually inject + 'help' => \$help # give help listing +) or pod2usage( 2 ); + +pod2usage( -msg => "inject one or many files into the IPP pipeline database", + -exitval => 2) if + defined $help; + +pod2usage( -msg => "Usage: $0 --telescope (name) --camera (name) [--workdir path] [--reduction class] [--dvodb db] [--tess_id tess] [--end_stage stage] [--label label] [--dbname dbname] (files)", + -exitval => 2 ) if + scalar @ARGV == 0; + +pod2usage( -msg => "Required options: --telescope (name) --camera (name)", + -exitval => 3) unless + defined $telescope and + defined $camera; + +my $pxinject = can_run('pxinject') or die "Can't find pxinject\n"; + +# if workdir is not defined, assign the current path +# XXX we need to handle relative paths for workdir (not allowed) +if (! $workdir) { + $workdir = File::Spec->rel2abs( "." ); +} + +my $num = 0; +foreach my $file ( @ARGV ) { + # check for file existence + if (! -e $file) { die "file $file not found\n"; } + my $absfile = File::Spec->rel2abs( $file ); + inject($absfile, $workdir, $dbname, $telescope, $camera); + $num ++; +} + +print "$num files injected.\n"; + +sub inject +{ + my $absfile = shift; # absolute path for this file + my $workdir = shift; # absolute path for output directory + my $dbname = shift; # IPP database to use + my $telescope = shift; # user-specified telescope + my $camera = shift; # user-specified camera + + # XXX provide an option for an alternative extension + my ( $vol, $path, $name ) = File::Spec->splitpath( $absfile ); + my ( $exp_name ) = $name =~ /(.*)\.(fits|fit|fts)(|.gz)/; + + my $relfile = $ipprc->convert_filename_relative( $absfile ); + + # the telescope, instrument, and exp_name used here are temporary : register replaces them with the true values + my $command_exp = "$pxinject -newExp"; + $command_exp .= " -tmp_exp_name $exp_name"; + $command_exp .= " -tmp_inst $camera"; + $command_exp .= " -tmp_telescope $telescope"; + $command_exp .= " -workdir $workdir"; + $command_exp .= " -reduction $reduction" if defined $reduction; + $command_exp .= " -dvodb $dvo_db" if defined $dvo_db; + $command_exp .= " -tess_id $tess_id" if defined $tess_id; + $command_exp .= " -end_stage $end_stage" if defined $end_stage; + $command_exp .= " -label $label" if defined $label; + $command_exp .= " -dbname $dbname" if defined $dbname; + + my $exp_id = 0; + unless ($no_op) { + my ( $success_exp, $error_code_exp, $full_buf_exp, $stdout_buf_exp, $stderr_buf_exp ) = + run( command => $command_exp, verbose => 1 ); + die "Unable to inject $exp_name: $error_code_exp\n" if not $success_exp; + + my @line = split(/\s+/, $$stdout_buf_exp[0]); # The output line, containing the exposure tag + $exp_id = $line[2]; # The exposure tag + } else { + print "skipping command: $command_exp\n"; + } + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_imfile = "$pxinject -newImfile"; + $command_imfile .= " -exp_id $exp_id"; + $command_imfile .= " -tmp_class_id fpa"; + $command_imfile .= " -uri $relfile"; + $command_imfile .= " -dbname $dbname" if defined ($dbname); + + unless ($no_op) { + my ( $success_imfile, $error_code_imfile, $full_buf_imfile, $stdout_buf_imfile, $stderr_buf_imfile ) = run( command => $command_imfile, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code_imfile\n" if not $success_imfile; + } else { + print "skipping command: $command_imfile\n"; + } + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_update = "$pxinject -updatenewExp"; + $command_update .= " -exp_id $exp_id"; + $command_update .= " -state run"; + $command_update .= " -dbname $dbname" if defined ($dbname); + + unless ($no_op) { + my ( $success_update, $error_code_update, $full_buf_update, $stdout_buf_update, $stderr_buf_update ) = run( command => $command_update, verbose => 1 ); + die "Unable to update $exp_name: $error_code_update\n" if not $success_update; + } else { + print "skipping command: $command_update\n"; + } + + return 1; +} + + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_inject_mosaic.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_inject_mosaic.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_inject_mosaic.pl (revision 22161) @@ -0,0 +1,168 @@ +#!/usr/bin/env perl + +# this program injects a set of multi-file exposures into the db. the +# program takes a list of base exposure names and injects all files +# associated with the exposure. It constructs the expected filenames +# from the exposure tag and rules for the camera. This program is for +# the test only since it requires too much information at the inject +# stage. use 'ipp_serial_inject.pl' for single-file images and +# 'ipp_serial_inject_split.pl' for multiple file images in split +# format + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Config qw( caturi ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($camera, # Camera used + $telescope, # Telescope used + $tess_id, # Tessellation identifier + $dbname, # Database name + $workdir, # Working directory + $path, # Path to data + ); +GetOptions( + 'camera|c=s' => \$camera, + 'telescope|t=s' => \$telescope, + 'workdir=s' => \$workdir, + 'tess_id=s' => \$tess_id, + 'path=s' => \$path, + 'dbname=s' => \$dbname, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --camera --telescope --workdir --path --dbname", + -exitval => 3, + ) unless defined $camera + and defined $telescope + and defined $workdir + and defined $path + and defined $dbname; + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +# Look for programs we need +my $missing_tools; +my $pxinject = can_run('pxinject') or (warn "Can't find pxinject" and $missing_tools = 1); + +if (scalar @ARGV == 0) { + die "No exposures provided.\n"; +} + +# Inject new data into the database +my @classes; # Names of the classes +my @files; # What to add to the filename for each class +my $imfiles; +my $add_dir; # Add directory name to get file name? +if ($camera eq "MEGACAM") { + for (my $i = 0; $i < 36; $i++) { + push @classes, sprintf("ccd%02d", $i); + push @files, sprintf(".ccd%02d", $i); + } +} elsif ($camera eq "MCSHORT") { + @classes = ( 'ccd12', 'ccd13', 'ccd14', 'ccd21', 'ccd22', 'ccd23' ); + @files = ( '.ccd12', '.ccd13', '.ccd14', '.ccd21', '.ccd22', '.ccd23' ); +} elsif ($camera eq "CTIO_MOSAIC2") { + @classes = (); + @files = (); +} elsif ($camera eq "TC3") { + @classes = ( 'CCID58-1-06b2', 'CCID45-1-14A', 'CCID45-1-11A', 'CCID45-1-22A', + 'CCID45-1-04C', 'CCID45-1-13A', 'CCID45-1-05A', 'CCID45-1-19A' ); + @files = ( '00', '01', '10', '11', '20', '21', '30', '31' ); + $add_dir = 1; +} elsif ($camera eq "SIMMOSAIC") { + @classes = ( 'Chip00', 'Chip01', 'Chip10', 'Chip11' ); + @files = ( '.Chip00', '.Chip01', '.Chip10', '.Chip11' ); +} elsif ($camera eq "SIMTEST") { + @classes = (); + @files = (); +} elsif ($camera eq "GPC1") { + for (my $i = 0; $i < 8; $i++) { + for (my $j = 0; $j < 8; $j++) { + if (($i == 0 or $i == 7) and ($j == 0 or $j == 7)) { + # Excluding corner chips + next; + } + push @classes, "XY$i$j"; + push @files, "$i$j"; + } + } + $add_dir = 1; +} else { + die "Unrecognised camera name: $camera.\nDid you mean to use ipp_serial_inject.pl?\n"; +} + +foreach my $exp_name ( @ARGV ) { + my $command = "$pxinject -newExp -tmp_exp_name $exp_name -tmp_inst $camera -tmp_telescope $telescope -workdir $workdir -tess_id $tess_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to inject $exp_name: $error_code\n" if not $success; + + my @line = split(/\s+/, $$stdout_buf[0]); # The output line, containing the exposure tag + my $exp_id = $line[2]; # The exposure tag + for (my $i = 0; $i < scalar @classes; $i++) { + my $class_id = $classes[$i]; + my $file_id = $files[$i]; + my $filename = $exp_name . $file_id . '.fits'; + $filename = caturi( $exp_name, $filename ) if defined $add_dir; + $filename = caturi( $path, $filename ); + + die "Unable to find file $filename" unless -f $ipprc->file_resolve( $filename ); + + $filename = $ipprc->convert_filename_relative( $filename ); + my $command = "$pxinject -newImfile -exp_id $exp_id -tmp_class_id $class_id -uri $filename"; # Command to run + $command .= " -dbname $dbname" if defined ($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to inject $exp_name $class_id: $error_code\n" if not $success; + } + + if (scalar @classes == 0) { + my $filename = $exp_name . '.fits'; + $filename = caturi( $exp_name, $filename ) if defined $add_dir; + $filename = caturi( $path, $filename ); + + die "Unable to find file $filename" unless -f $ipprc->file_resolve( $filename ); + + $filename = $ipprc->convert_filename_relative( $filename ); + my $command = "$pxinject -newImfile -exp_id $exp_id -tmp_class_id fpa -uri $filename"; # Command to run + $command .= " -dbname $dbname" if defined ($dbname); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code\n" if not $success; + } + + # Update the exposure to run + { + my $command = "$pxinject -updatenewExp -exp_id $exp_id -state run"; # Command to run + $command .= " -dbname $dbname" if defined ($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to activate $exp_name: $error_code\n" if not $success; + } + +} + +END { + my $status = $?; +system("sync") == 0 + or die "failed to execute sync: $!" ; +$? = $status; +} + + +__END__ + + Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_inject_split.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_inject_split.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_inject_split.pl (revision 22161) @@ -0,0 +1,138 @@ +#!/usr/bin/env perl + +# this program injects a list of multi-file exposures into the db. +# the program takes a list of directory names (dir), and injects all +# dir/*.fits. It takes the directory name as the exp_tag. the user +# supplies a temporary telescope and instrument name. these are used +# for informational purposes only until the registration step can +# determine the true telescope and instrument name from the image +# headers. + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use PS::IPP::Config; + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ($camera, $telescope, $workdir, $reduction, $dvo_db, $tess_id, $end_stage, $dbname, $help); +GetOptions('camera|c=s' => \$camera, # Camera used + 'telescope|t=s' => \$telescope, # Telescope used + 'workdir|w=s' => \$workdir, # working directory for output files + 'reduction=s' => \$reduction, # user-supplied camera name + 'dvodb=s' => \$dvo_db, # target dvo database + 'tess_id=s' => \$tess_id, # tessalation for warping + 'end_stage=s' => \$end_stage, # stop processing at this step + 'dbname|d=s' => \$dbname, # Database name + 'help' => \$help # give help listing +) or pod2usage( 2 ); + +pod2usage( -msg => "inject one or many image exposures (directories) into the IPP pipeline database", + -exitval => 2) if + defined $help; + +pod2usage( -msg => "Usage: $0 --telescope (name) --camera (name) [--workdir path] [--reduction class] [--dvodb db] [--tess_id tess] [--end_stage stage] [--dbname name] (files)", + -exitval => 2 ) if + scalar @ARGV == 0; + +pod2usage( + -msg => "Required options: --camera --telescope", + -exitval => 3) unless + defined $telescope and + defined $camera; + +# Look for programs we need +my $pxinject = can_run('pxinject') or die "Can't find pxinject\n"; + +# if workdir is not defined, assign the current path +# we need to handle relative paths for workdir (not allowed) +if (! $workdir) { + $workdir = File::Spec->rel2abs( "." ); +} + +my $num = 0; +foreach my $filedir ( @ARGV ) { + # check for filedir existence + if (! -e $filedir) { die "file dir $filedir not found\n"; } + my $absfiledir = File::Spec->rel2abs( $filedir ); + inject($absfiledir, $workdir, $dbname, $telescope, $camera); + $num ++; +} + +sub inject +{ + my $absfiledir = shift; # absolute path for this file + my $workdir = shift; # absolute path for output directory + my $dbname = shift; # IPP database to use + my $telescope = shift; # user-specified telescope + my $camera = shift; # user-specified camera + + # XXX provide an option for an alternative extension + my @files = <$absfiledir/*.fits>; + my $Nfiles = scalar @files; + + print "absfiledir: $absfiledir\n"; + print "Nfiles: $Nfiles\n"; + + # the absfiledir is of the form path://PATH/data/foo/expname + my ( $vol, $path, $exp_name ) = File::Spec->splitpath( $absfiledir ); + + # the telescope, instrument, and exp_name used here are temporary : register replaces them with the true values + my $command_exp = "$pxinject -newExp"; + $command_exp .= " -tmp_exp_name $exp_name"; + $command_exp .= " -tmp_inst $camera"; + $command_exp .= " -tmp_telescope $telescope"; + $command_exp .= " -workdir $workdir"; + $command_exp .= " -reduction $reduction" if defined $reduction; + $command_exp .= " -dvodb $dvo_db" if defined $dvo_db; + $command_exp .= " -tess_id $tess_id" if defined $tess_id; + $command_exp .= " -end_stage $end_stage" if defined $end_stage; + $command_exp .= " -dbname $dbname" if defined $dbname; + + my ( $success_exp, $error_code_exp, $full_buf_exp, $stdout_buf_exp, $stderr_buf_exp ) = + run( command => $command_exp, verbose => 1 ); + die "Unable to inject $exp_name: $error_code_exp\n" if not $success_exp; + + my @line = split(/\s+/, $$stdout_buf_exp[0]); # The output line, containing the exposure tag + my $exp_id = $line[2]; # The exposure tag + + foreach my $absfile (@files) { + + my $relfile = $ipprc->convert_filename_relative( $absfile ); + + my ( $tmpvol, $tmppath, $filename ) = File::Spec->splitpath( $absfile ); + my ( $class_name ) = $filename =~ /(.*)\.fits/; + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_imfile = "$pxinject -newImfile"; + $command_imfile .= " -exp_id $exp_id"; + $command_imfile .= " -tmp_class_id $class_name"; + $command_imfile .= " -uri $relfile"; + $command_imfile .= " -dbname $dbname" if defined ($dbname); + + my ( $success_imfile, $error_code_imfile, $full_buf_imfile, $stdout_buf_imfile, $stderr_buf_imfile ) = run( command => $command_imfile, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code_imfile\n" if not $success_imfile; + } + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_update = "$pxinject -updatenewExp"; + $command_update .= " -exp_id $exp_id"; + $command_update .= " -state run"; + $command_update .= " -dbname $dbname" if defined ($dbname); + + my ( $success_update, $error_code_update, $full_buf_update, $stdout_buf_update, $stderr_buf_update ) = run( command => $command_update, verbose => 1 ); + die "Unable to update $exp_name: $error_code_update\n" if not $success_update; + + return 1; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_mops.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_mops.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_mops.pl (revision 22161) @@ -0,0 +1,123 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use DBI; + +use IPC::Cmd 0.36 qw( can_run run ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use Data::Dumper; +use File::Temp qw( tempfile ); + +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config qw( caturi ); + +# Look for programs we need +my $missing_tools; +my $mops = can_run('ipp_mops_translate.pl') or (warn "Can't find ipp_mops_translate.pl" and $missing_tools = 1); +my $dsreg = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +my $ipprc = PS::IPP::Config->new; # IPP Configuration + +my ( $dbhost, # Database host + $dbname, # Database name + $dbuser, # Database user + $dbpass, # Database p/w + $camera, # Camera used + $outroot, # Output directory + $fileset, # File set + $verbose, # Verbose output? + $no_update, # Don't update state? + $no_op, # Don't do any operations? + $save_temps # Save temporary files? + ); + +GetOptions( + 'dbhost=s' => \$dbhost, + 'dbname=s' => \$dbname, + 'dbuser=s' => \$dbuser, + 'dbpass=s' => \$dbpass, + 'camera=s' => \$camera, + 'outroot=s' => \$outroot, + 'fileset=s' => \$fileset, + 'verbose' => \$verbose, + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'save-temps' => \$save_temps, # Save temporary files? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --dbhost --dbname --dbuser --dbpass --camera --outroot --fileset", + -exitval => 3) + unless defined $dbhost + and defined $dbname + and defined $dbuser + and defined $dbpass + and defined $camera + and defined $outroot + and defined $fileset; + +$ipprc->define_camera($camera); + +my $dbsrc = 'DBI:mysql:database=' . $dbname . ';host=' . $dbhost . + ';mysql_socket=/var/run/mysqld/mysqld.sock'; +my $db = DBI->connect($dbsrc, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1 } ) or + die "Unable to connect to database: $DBI::errstr"; + +my $sql = "SELECT exp_id, diff_id, skycell_id, warp_id, diffSkyfile.path_base AS diff_path_base, warpSkyfile.path_base AS warp_path_base FROM diffSkyfile JOIN diffRun using(diff_id) JOIN diffInputSkyfile USING(diff_id,tess_id,skycell_id) JOIN warpRun USING(warp_id,tess_id) JOIN warpSkyfile USING(warp_id,skycell_id,tess_id) JOIN fakeRun using(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) WHERE diffSkyfile.fault = 0 AND rawExp.camera = \'$camera\';"; + +my $rows = $db->selectall_arrayref( $sql, { Slice => {} } ) or die "Unable to execute SQL: $DBI::errstr"; +$db->disconnect; + +print "Selected " . scalar @$rows . " rows.\n"; + +$ipprc->outroot_prepare( $outroot ); +my $outrootResolved = $ipprc->file_resolve( $outroot ); +my ($dsFile, $dsName) = tempfile( "$outrootResolved.dslist.XXXX", UNLINK => !$save_temps); + +foreach my $row ( @$rows ) { + my $exp_id = $row->{exp_id}; + my $diff_id = $row->{diff_id}; + my $skycell_id = $row->{skycell_id}; + my $warp_id = $row->{warp_id}; + my $diff_base = $row->{diff_path_base}; + my $warp_base = $row->{warp_path_base}; + + my $input = $ipprc->filename("PSPHOT.OUTPUT", $diff_base); + die "Can't find $input\n" unless $ipprc->file_exists($input); + $input = $ipprc->file_resolve($input); + + my $skycell = $ipprc->filename("SKYCELL.TEMPLATE", $warp_base); + die "Can't find $skycell\n" unless $ipprc->file_exists($skycell); + $skycell = $ipprc->file_resolve($skycell); + + my $output = caturi( $outroot, "mops.$exp_id.$skycell_id.$diff_id.fits" ); + $output = $ipprc->file_resolve($output); + + unless ($no_op) { + $ipprc->file_prepare($output); + my $command = "$mops --input $input --extname SkyChip.psf --skycell $skycell --output $output"; + my $success = run( command => $command, verbose => $verbose ); + die "Couldn't translate $input\n" unless $success; + } + + # format: filename|filesize|md5sum|filetype| + # note: since we omit filesize and md5sum, dsreg will calculate them + print $dsFile "${output}|||ipp-mops|\n"; +} +close $dsFile; + +# Register new files with the data store +unless ($no_update) { + my $command = "$dsreg --add $fileset --product mops_transient_detections --type MOPS_TRANSIENT_DETECTIONS --list $dsName --copy --abspath --dbname DataStore"; + my $success = run( command => $command, verbose => $verbose ); + die "Couldn't register files with data store.\n" unless $success; +} + + +__END__ + Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_register.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_register.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_register.pl (revision 22161) @@ -0,0 +1,118 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dbname, # Database name to use + ); +GetOptions( + 'dbname|d=s' => \$dbname, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $register_imfile = can_run('register_imfile.pl') or (warn "Can't find register_imfile.pl" and $missing_tools = 1); +my $register_exp = can_run('register_exp.pl') or (warn "Can't find register_exp.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Phase 0 imfile processing +{ + my $command = "$regtool -pendingimfile -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get phase 0 imfile list: $error_code\n" if not $success; + + my @whole = split /\n/, join( '', @$stdout_buf ); + my @single = (); + + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from regtool.\n"; + + foreach my $item (@$list) { + my $exp_id = $item->{exp_id}; + my $exp_name = $item->{tmp_exp_name}; + my $class_id = $item->{tmp_class_id}; + my $uri = $item->{uri}; + + my $command = "$register_imfile --exp_id $exp_id --tmp_class_id $class_id --tmp_exp_name $exp_name --uri $uri --dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do phase 0 imfile processing on $exp_name $class_id: $error_code\n" if not $success; + } + + + @single = (); + + } + } +} + +# Phase 0 exposure processing +{ + my $command = "$regtool -pendingexp -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get phase 0 exposure list: $error_code\n" if not $success; + + my @whole = split /\n/, join( '', @$stdout_buf ); + my @single = (); + + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from regtool.\n"; + + foreach my $item (@$list) { + my $exp_tag = $item->{tmp_exp_name} . '.' . $item->{exp_id}; + my $exp_id = $item->{exp_id}; + + my $command = "$register_exp --exp_tag $exp_tag --exp_id $exp_id --dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do phase 0 exposure processing on $exp_tag: $error_code\n" if not $success; + } + + @single = (); + + } + } +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +__END__ + + Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_stack.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_stack.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_stack.pl (revision 22161) @@ -0,0 +1,99 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use File::Basename; + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, # Database name to use + $verbose, # Verbose operations? + $workdir_global, # Global working directory + $no_op, # No operations? + $no_update, # No updating? + $save_temps, # Save temporary files? + ); +GetOptions( + 'dbname=s' => \$dbname, + 'verbose' => \$verbose, + 'workdir' => \$workdir_global, + 'no-op' => \$no_op, + 'no-update' => \$no_update, + 'save-temps' => \$save_temps, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $stacktool = can_run('stacktool') or + (warn "Can't find difftool" and $missing_tools = 1); +my $stack_skycell = can_run('stack_skycell.pl') or + (warn "Can't find stack_skycell.pl" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + + +# Image stacking +{ + my $list; + my $command = "$stacktool -tosum -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get list of stacks: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from stacktool.\n"; + + foreach my $item (@$list) { + my $stack_id = $item->{stack_id}; + my $workdir = $item->{workdir}; + my $tess_id = basename($item->{tess_id}); + my $skycell_id = $item->{skycell_id}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, $tess_id, $skycell_id, "$tess_id.$skycell_id.stk.$stack_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$stack_skycell --stack_id $stack_id --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + $command .= " --save-temps" if defined $save_temps; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); +### die "Unable to do stack for $stack_id: $error_code\n" if not $success; + } +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_warp.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_warp.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_serial_warp.pl (revision 22161) @@ -0,0 +1,129 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use File::Basename; + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, # Database name to use + $verbose, # Verbose operations? + $workdir_global, # Global working directory + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname=s' => \$dbname, + 'verbose' => \$verbose, + 'workdir' => \$workdir_global, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $warptool = can_run('warptool') or + (warn "Can't find warptool" and $missing_tools = 1); +my $warp_skycell = can_run('warp_skycell.pl') or + (warn "Can't find warp_skycell.pl" and $missing_tools = 1); +my $warp_overlap = can_run('warp_overlap.pl') or + (warn "Can't find warp_overlap.pl" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + + +# Calculate overlaps +{ + my $list; + my $command = "$warptool -tooverlap -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get warps for which to calculate overlaps: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from warptool.\n"; + + foreach my $item (@$list) { + my $warp_id = $item->{warp_id}; + my $cam_id = $item->{cam_id}; + my $workdir = $item->{workdir}; + my $camera = $item->{camera}; + my $tess_id = $item->{tess_id}; + + my $command = "$warp_overlap --warp_id $warp_id --camera $camera --tess_id $tess_id --dbname $dbname"; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get warp overlaps on $warp_id: $error_code\n" if not $success; + } +} + + +# Warping proper +{ + my $list; + my $command = "$warptool -towarped -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get warps for warping: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from warptool.\n"; + + foreach my $item (@$list) { + my $warp_id = $item->{warp_id}; + my $skycell_id = $item->{skycell_id}; + my $tess_id = basename($item->{tess_id}); + my $cam_id = $item->{cam_id}; + my $workdir = $item->{workdir}; + my $camera = $item->{camera}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, $tess_id, $skycell_id, "$tess_id.$skycell_id.wrp.$warp_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$warp_skycell --warp_id $warp_id --skycell_id $skycell_id --tess_id $tess_id --camera $camera --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do warp processing on $warp_id,$skycell_id: $error_code\n" if not $success; + } +} + + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_simulation_data.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_simulation_data.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipp_simulation_data.pl (revision 22161) @@ -0,0 +1,215 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use Math::Trig; +use File::Spec; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($name, # Base name for output images + $camera, # Name of camera to use + $telescope, # Telescope name + $tess_id, # Tessellation identifier + $dbname, # Database name + $path, # Path to data + $workdir, # Working directory for data + $no_cal, # Don't produce calibration files + $no_update # Don't update the database + ); + +GetOptions( + 'name=s' => \$name, + 'camera=s' => \$camera, + 'telescope=s' => \$telescope, + 'tess_id=s' => \$tess_id, + 'dbname=s' => \$dbname, + 'path=s' => \$path, + 'workdir=s' => \$workdir, + 'no-cal' => \$no_cal, + 'no-update' => \$no_update, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --name --path --camera --telescope --dbname", + -exitval => 3, + ) unless + defined $name and + defined $path and + defined $camera and + defined $telescope and + defined $dbname; + +$workdir = $path if not defined $workdir; + +# Look for programs we need +my $missing_tools; +my $ppSim = can_run('ppSim') + or (warn "Can't find ppSim" and $missing_tools = 1); +my $inject = can_run('ipp_serial_inject_mosaic.pl') + or (warn "Can't find ipp_serial_inject_mosaic.pl" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Number of bias images +use constant BIAS => 20; +# Dark exposure times +use constant DARK => [ 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 300, 300, 300, 300, 300, 300, 300, 300, 300, 300 ]; +# Flat-field image characteristics +use constant FLAT => [ + { + filter => 'r', + exptime => [ 0.1, 0.1, 0.1, 0.1, 0.1, 0.5, 1, 2, 5, 10, 20, 20, 20, 20, 20 ], + }, + { + filter => 'i', + exptime => [ 20, 20, 20, 20, 20 ], + } + ]; +# Object image characteristics +use constant OBJECT => [ + { + filter => 'r', + exptime => [ 5, 10, 10, 10, 10, 10, 240 ], + seeing => [ 0.8, 0.4, 0.6, 0.8, 1.2, 1.5, 0.7 ], + ra => 150.119167, + dec => 2.205833, + pa => 0, + zp => 25.15, + sky => 20.86, + dither => 40, + }, + { + filter => 'i', + exptime => [ 5, 30, 30, 30, 30, 30, 240 ], + seeing => [ 0.8, 0.4, 0.6, 0.8, 1.2, 1.5, 0.7 ], + ra => 150.119167, + dec => 2.205833, + pa => 0, + zp => 25.00, + sky => 20.15, + dither => 40, + }, + ]; +use constant SCALE => 0.2; # Plate scale + +############################################################################################################# +### Now do the work +############################################################################################################# + +my $counter = 0; +my $tessellation = (defined $tess_id) ? "--tess_id $tess_id" : ""; # Additional switch for tessellation + +# Generate bias images +for (my $i = 0; $i < BIAS; $i++) { + my $basename; # Output base filename + ( $basename, $counter ) = filename( $name, $counter ); + my $filename = caturi( $path, $basename ); + unless ($no_cal) { + run( command => "$ppSim -camera $camera -type BIAS $filename", + verbose => 1 ) or die "Unable to run ppSim"; + unless ($no_update) { + run( command => "$inject --camera $camera --telescope $telescope --path $path " . + "$tessellation --workdir $workdir --dbname $dbname $basename", + verbose => 1 ) or die "Unable to inject file."; + } + } +} + +# Generate dark images +foreach my $exptime ( @{DARK()} ) { + my $basename; # Output base filename + ( $basename, $counter ) = filename( $name, $counter ); + my $filename = caturi( $path, $basename ); + unless ($no_cal) { + run ( command => "$ppSim -camera $camera -type DARK -exptime $exptime $filename", + verbose => 1 ) or die "Unable to run ppSim"; + unless ($no_update) { + run( command => "$inject --camera $camera --telescope $telescope --path $path " . + "$tessellation --workdir $workdir --dbname $dbname $basename", + verbose => 1 ) or die "Unable to inject file."; + } + } +} + +# Generate flat images +foreach my $set ( @{FLAT()} ) { + my $filter = $set->{filter}; # Name of filter + foreach my $exptime ( @{$set->{exptime}} ) { + my $basename; # Output base filename + ( $basename, $counter ) = filename( $name, $counter ); + my $filename = caturi( $path, $basename ); + unless ($no_cal) { + run( command => "$ppSim -camera $camera -type FLAT -filter $filter -exptime $exptime $filename", + verbose => 1 ) or die "Unable to run ppSim"; + unless ($no_update) { + run( command => "$inject --camera $camera --telescope $telescope --path $path " . + "$tessellation --workdir $workdir --dbname $dbname $basename", + verbose => 1 ) or die "Unable to inject file."; + } + } + } +} + +# Generate object images +foreach my $set ( @{OBJECT()} ) { + my $filter = $set->{filter}; # Name of filter + my $ra0 = $set->{ra}; # Base Right Ascension (deg) + my $dec0 = $set->{dec}; # Base Declination (deg) + my $pa = $set->{pa}; # Position angle (deg) + my $zp = $set->{zp}; # Zero point + my $scale = SCALE(); # Plate scale (arcsec/pix) + my $sky = 10**( -0.4 * ( $set->{sky} - $zp ) ) * $scale**2; # Sky background (counts/s) + my $dither = $set->{dither} / 3600; # Dither size (deg) + + for (my $i = 0; $i < scalar @{$set->{exptime}}; $i++) { + my $exptime = ${$set->{exptime}}[$i]; # Exposure time + my $seeing = ${$set->{seeing}}[$i]; # Seeing (pix) + my $ra = $ra0 + (2*rand() - 1) * $dither * cos(deg2rad($dec0)); # RA with dither + my $dec = $dec0 + (2*rand() - 1) * $dither; # Dec with dither + + my $basename; # Output base filename + ( $basename, $counter ) = filename( $name, $counter ); + my $filename = caturi( $path, $basename ); + run( command => "$ppSim -camera $camera -type OBJECT -filter $filter -exptime $exptime " . + "-skyrate $sky -ra $ra -dec $dec -pa $pa -scale $scale -zp $zp -seeing $seeing $filename", + verbose => 1 ) or die "Unable to run ppSim"; + unless ($no_update) { + run( command => "$inject --camera $camera --telescope $telescope --path $path " . + "$tessellation --workdir $workdir --dbname $dbname $basename", + verbose => 1 ) or die "Unable to inject file."; + } + } +} + + +### Pau. + + +# Generate a filename from the base name and counter +sub filename +{ + my $base = shift; # Base name + my $num = shift; # Number + my $workdir = shift; # Working directory + my $name = sprintf("$base%04d", $num); + $num++; + return ( $name, $num ); +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipprc.txt =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipprc.txt (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/ipprc.txt (revision 22161) @@ -0,0 +1,16 @@ + +these scripts use the following library functions to work with the IPP Config system: + +$ipprc->convert_filename_absolute( $tess_dir ); +$ipprc->convert_filename_relative( $absfile ); +$ipprc->define_camera($camera); +$ipprc->extname_rule("CMF.HEAD", $class_id); # MEF psastro output +$ipprc->file_create_append( $logName ); +$ipprc->file_exists( $skyfile ); +$ipprc->file_prepare( "$exp_tag/$exp_tag.detproc.$det_id", $workdir, $input_uri ); +$ipprc->file_resolve ($path_base); +$ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n"); +$ipprc->outroot_prepare($outroot); +$ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE_' . uc($det_type)); # Recipe to use +$ipprc->rejection( $name, $det_type, $filter ); +$ipprc->tessellation_catdir( $tess_id ); # Tessellation catdir for DVO Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/isp_trans.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/isp_trans.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/isp_trans.pl (revision 22161) @@ -0,0 +1,30 @@ +#!/usr/bin/env perl +# basic ISP transmission analysis: + +if (@ARGV != 1) { die "USAGE: isp_trans.pl (input.fits)\n"; } +$input = $ARGV[0]; + +# for input file /path/foo.fits, use /path/foo for output + +@words = split ('\.', $input); +if (@words > 1) { pop @words; } +$output = join (".", @words); + +# use constant RECIPE => 'PPIMAGE_OBDSFRA'; # Recipe to use +$RECIPE_PPIMAGE = 'PPIMAGE_OA'; # Recipe to use (switch to OBDSFRA when detrend images are ready) +$RECIPE_PSPHOT = 'PSPHOT.SUMMIT'; +$CALDIR = '/data/alala.0/ipp/ippRefs/catdir.synth.bright'; # source of photometric calibration data +$IMTABLE = 'images.dat'; # source of photometric calibration data + +vsystem ("ppImage -file $input $output -recipe PPIMAGE $RECIPE_PPIMAGE -recipe PSPHOT $RECIPE_PSPHOT"); +if ($status) { die "failure running ppImage\n"; } + +vsystem ("addstar -incal -image -D CAMERA isp -D IMAGE_TABLE $IMTABLE -D CATDIR $CALDIR $output.smf"); +if ($status) { die "failure getting calibration from addstar\n"; } + +sub vsystem { + print STDERR "@_\n"; + my $status = system ("@_"); + $status; +} + Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/magic_mask.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/magic_mask.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/magic_mask.pl (revision 22161) @@ -0,0 +1,171 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use Astro::FITS::CFITSIO qw( :constants ); +Astro::FITS::CFITSIO::PerlyUnpacking(1); + +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ($magic_id, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $redirect); +GetOptions( + 'magic_id=s' => \$magic_id, # Magic identifier + 'camera=s' => \$camera, # Camera name + 'dbname=s' => \$dbname, # Database name + 'outroot=s' => \$outroot, # Output root name + 'save-temps' => \$save_temps, # Save temporary files? + 'verbose' => \$verbose, # Print stuff? + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, # Redirect output? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --magic_id --camera --outroot", + -exitval => 3) unless + defined $magic_id and + defined $camera and + defined $outroot; + +$ipprc->define_camera($camera); + +my $logDest = $ipprc->filename("LOG.EXP", $outroot, $magic_id) or + &my_die("Missing entry from camera config", $magic_id, $PS_EXIT_CONFIG_ERROR); +$ipprc->redirect_output($logDest) if $redirect; + +# Look for programs we need +my $missing_tools; +my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1); +my $streaks = can_run('RemoveStreaks') or (warn "Can't find RemoveStreaks" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +### Get a list of inputs +my $inputs; # List of inputs +{ + my $command = "$magictool -inputs -magic_id $magic_id -node root"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magictool -inputs: $error_code", $magic_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $magic_id, $PS_EXIT_PROG_ERROR); + + $inputs = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $magic_id, $PS_EXIT_PROG_ERROR); +} + +### Do the heavy lifting +{ + my $command = "$streaks --streaks"; # Command to execute + my @basenames; # List of base names + + # Concatenate the names + foreach my $node (@$inputs) { + push @basenames, $ipprc->file_resolve( $node->{path_base} ); + } + $command .= ' ' . join(' ', @basenames); + + unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform RemoveStreaks: $error_code", $magic_id, $error_code); + } + + foreach my $basename (@basenames) { + file_check( $basename . '.mask' ); + } + } else { + print "Skipping command: $command\n"; + } +} + + + +### Input mask into database +{ + my $command = "$magictool -addmask"; + $command .= " -magic_id $magic_id"; + $command .= " -uri $outroot"; + $command .= " -dbname $dbname" if defined $dbname; + + # Add the processed file to the database + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magictool -addmask: $error_code", $magic_id, $error_code); + } + } else { + print "Skipping command: $command\n"; + } +} + + + +### Pau. + +sub file_check +{ + my $file = shift; # Name of file + &my_die("Unable to find output file: $file", $magic_id, $PS_EXIT_SYS_ERROR) unless + $ipprc->file_exists($file); +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $magic_id = shift; # Magic identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $magic_id and not $no_update) { + my $command = "$magictool -addmask"; + $command .= " -magic_id $magic_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/magic_process.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/magic_process.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/magic_process.pl (revision 22161) @@ -0,0 +1,190 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use Astro::FITS::CFITSIO qw( :constants ); +Astro::FITS::CFITSIO::PerlyUnpacking(1); + +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ($magic_id, $node, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $redirect); +GetOptions( + 'magic_id=s' => \$magic_id, # Magic identifier + 'node=s' => \$node, # Node name + 'camera=s' => \$camera, # Camera name + 'dbname=s' => \$dbname, # Database name + 'outroot=s' => \$outroot, # Output root name + 'save-temps' => \$save_temps, # Save temporary files? + 'verbose' => \$verbose, # Print stuff? + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, # Redirect output? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --magic_id --camera --node --outroot", + -exitval => 3) unless + defined $magic_id and + defined $node and + defined $camera and + defined $outroot; + +$ipprc->define_camera($camera); + +my $logDest = $ipprc->filename("LOG.EXP", $outroot, $magic_id) or + &my_die("Missing entry from camera config", $magic_id, $PS_EXIT_CONFIG_ERROR); +$ipprc->redirect_output($logDest) if $redirect; + +# Look for programs we need +my $missing_tools; +my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1); +my $streaks = can_run('RemoveStreaks') or (warn "Can't find RemoveStreaks" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +### Get a list of inputs +my $inputs; # List of inputs +{ + my $command = "$magictool -inputs -magic_id $magic_id -node $node"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magictool -inputs: $error_code", $magic_id, $node, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $magic_id, $node, $PS_EXIT_PROG_ERROR); + + $inputs = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $magic_id, $node, $PS_EXIT_PROG_ERROR); +} + + +### Do the heavy lifting +my @basenames; # Base names of inputs +{ + my $command; # Command to execute + if (scalar @$inputs == 1 and $node ne "root") { + # Leaf node: 'detect' stage + $command = "$streaks --detect"; + + my $node = $$inputs[0]; # Input node + push @basenames, $ipprc->file_resolve( $node->{path_base} ); + } else { + # Branch node: 'merge' stage + $command = "$streaks --merge"; + + # Concatenate the names + foreach my $node (@$inputs) { + push @basenames, $ipprc->file_resolve( $node->{path_base} ); + } + } + + $command .= ' ' . join(' ', @basenames); + + unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform RemoveStreaks: $error_code", $magic_id, $node, $error_code); + } + + foreach my $basename (@basenames) { + file_check( $basename . '.clusters' ); + file_check( $basename . '.streaks' ); + file_check( $basename . '_hough.fits' ); + } + } else { + print "Skipping command: $command\n"; + } +} + + + +### Input result into database +{ + my $command = "$magictool -addresult"; + $command .= " -magic_id $magic_id"; + $command .= " -node $node"; + $command .= " -uri $outroot"; + $command .= " -dbname $dbname" if defined $dbname; + + # Add the processed file to the database + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magictool -addresult: $error_code", $magic_id, $node, $error_code); + } + } else { + print "Skipping command: $command\n"; + } +} + + + +### Pau. + +sub file_check +{ + my $file = shift; # Name of file + &my_die("Unable to find output file: $file", $magic_id, $node, $PS_EXIT_SYS_ERROR) unless + $ipprc->file_exists($file); +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $magic_id = shift; # Magic identifier + my $node = shift; # Node name + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $magic_id and defined $node and not $no_update) { + my $command = "$magictool -addresult"; + $command .= " -magic_id $magic_id"; + $command .= " -node $node"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/magic_tree.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/magic_tree.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/magic_tree.pl (revision 22161) @@ -0,0 +1,332 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use Astro::FITS::CFITSIO qw( :constants ); +Astro::FITS::CFITSIO::PerlyUnpacking(1); + +use Math::Trig; +use File::Temp qw( tempfile ); +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +use constant MAX_FIELDS => 4; # Maximum number of fields to be in a node + +# Parse the command-line arguments +my ($magic_id, $tess_id, $camera, $ra0, $dec0, $dbname, $outroot, + $save_temps, $verbose, $no_update, $no_op, $redirect); +GetOptions( + 'magic_id=s' => \$magic_id, # Magic identifier + 'tess_id=s' => \$tess_id, # Tessellation identifier + 'camera=s' => \$camera, # Camera name + 'ra=f' => \$ra0, # Boresight right ascension, radians + 'dec=f' => \$dec0, # Boresight declination, radians + 'dbname=s' => \$dbname, # Database name + 'outroot=s' => \$outroot, # Output root name + 'save-temps' => \$save_temps, # Save temporary files? + 'verbose' => \$verbose, # Print stuff? + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, # Redirect output? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --magic_id --camera --tess_id --ra --dec --outroot", + -exitval => 3) unless + defined $magic_id and + defined $tess_id and + defined $ra0 and + defined $dec0 and + defined $camera and + defined $outroot; + +$ipprc->define_camera($camera); + +my $logDest = $ipprc->filename("LOG.EXP", $outroot, $magic_id) or + &my_die("Missing entry from camera config", $magic_id, $PS_EXIT_CONFIG_ERROR); +$ipprc->redirect_output($logDest) if $redirect; + +# Look for programs we need +my $missing_tools; +my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +### Get a list of skycells +my @skycells; # List of skycells +{ + my $command = "$magictool -inputskyfile -magic_id $magic_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magictool -inputfile: $error_code", $magic_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $magic_id, $PS_EXIT_PROG_ERROR); + + my $inputs = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $magic_id, $PS_EXIT_PROG_ERROR); + + foreach my $input ( @$inputs ) { + push @skycells, $input->{node}; # NB: Storing the skycell_id in magicInputSkyfile.node + } +} + +### For each skycell, project centre of skycell onto tangent plane of boresight +my @fields; +foreach my $skycell_id ( @skycells ) { + my $skyfile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id ); + $ipprc->skycell_file($tess_id, $skycell_id, $skyfile, $verbose) or &my_die("Unable to generate skycells $skycell_id", $magic_id, $PS_EXIT_PROG_ERROR); + my $skyfileResolved = $ipprc->file_resolve( $skyfile ); + my ($header, $status) = Astro::FITS::CFITSIO::fits_read_header( $skyfileResolved ); + &my_die("Unable to read skycell header: $status", $magic_id, $PS_EXIT_SYS_ERROR) if $status; + + # Get the useful header keywords + my $naxis1 = $$header{'NAXIS1'} or &my_die("Can't find NAXIS1", $magic_id, $PS_EXIT_SYS_ERROR); + my $naxis2 = $$header{'NAXIS2'} or &my_die("Can't find NAXIS2", $magic_id, $PS_EXIT_SYS_ERROR); + my $ctype1 = $$header{'CTYPE1'} or &my_die("Can't find CTYPE1", $magic_id, $PS_EXIT_SYS_ERROR); + my $ctype2 = $$header{'CTYPE2'} or &my_die("Can't find CTYPE2", $magic_id, $PS_EXIT_SYS_ERROR); + my $cdelt1 = $$header{'CDELT1'} or &my_die("Can't find CDELT1", $magic_id, $PS_EXIT_SYS_ERROR); + my $cdelt2 = $$header{'CDELT2'} or &my_die("Can't find CDELT2", $magic_id, $PS_EXIT_SYS_ERROR); + my $crval1 = $$header{'CRVAL1'} or &my_die("Can't find CRVAL1", $magic_id, $PS_EXIT_SYS_ERROR); + my $crval2 = $$header{'CRVAL2'} or &my_die("Can't find CRVAL2", $magic_id, $PS_EXIT_SYS_ERROR); + my $crpix1 = $$header{'CRPIX1'} or &my_die("Can't find CRPIX1", $magic_id, $PS_EXIT_SYS_ERROR); + my $crpix2 = $$header{'CRPIX2'} or &my_die("Can't find CRPIX2", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc11 = $$header{'PC001001'} or &my_die("Can't find PC001001", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc12 = $$header{'PC001002'} or &my_die("Can't find PC001002", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc21 = $$header{'PC002001'} or &my_die("Can't find PC002001", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc22 = $$header{'PC002002'} or &my_die("Can't find PC002002", $magic_id, $PS_EXIT_SYS_ERROR); + my $crota1 = $$header{'CROTA1'}; + my $crota2 = $$header{'CROTA2'}; + + &my_die("Unexpected projection: $ctype1 and $ctype2.", $magic_id, $PS_EXIT_SYS_ERROR) unless + $ctype1 =~ /^\'RA---TAN\s*\'$/ and $ctype2 =~ /^\'DEC--TAN\s*\'$/; + &my_die("Can't determine size of skycell ($naxis1,$naxis2)", $magic_id, $PS_EXIT_SYS_ERROR) unless + $naxis1 > 0 and $naxis2 > 0; + &my_die("Can't determine scale of skycell ($cdelt1,$cdelt2)", $magic_id, $PS_EXIT_SYS_ERROR) if + not defined $cdelt1 or $cdelt1 == 0 or not defined $cdelt2 or $cdelt2 == 0; + &my_die("We don't know how to handle rotations ($crota1,$crota2)", $magic_id, $PS_EXIT_SYS_ERROR) + if defined $crota1 or defined $crota2; + + # Relative coordinates of centre of the field + my $x = $naxis1 - $crpix1; + my $y = $naxis2 - $crpix2; + + # Coordinates on tangent plane + my $xi = $pc11 * ($x) + $pc12 * ($y); + my $eta = $pc21 * ($x) + $pc22 * ($y); + $xi *= $cdelt1; + $eta *= $cdelt2; + + # Coordinates on rotated celestial sphere + my $phi = atan2($eta,$xi) + pi/2; + my $theta = atan(180 / pi / sqrt($xi**2 + $eta**2)); + + # Coordinates on celestial sphere + $crval1 = deg2rad($crval1); + $crval2 = deg2rad($crval2); + my $ra = $crval1 + atan2(cos($theta) * sin($phi), + sin($theta) * cos($crval2) + cos($theta) * sin($crval2) * cos($phi)); + my $dec = asin(sin($theta) * sin($crval2) - cos($theta) * cos($crval2) * cos($phi)); + + # Rotate to boresight + my $phi_new = atan2(cos($dec) * sin($ra - $ra0), + sin($dec) * cos($dec0) + cos($dec) * sin($dec0) * cos($ra - $ra0)); + my $theta_new = asin(sin($dec) * sin($dec0) - cos($dec) * cos($dec0) * cos($ra - $ra0)); + + # Project + my $rad = 180 / pi * cot($theta_new); + my $xi_new = $rad * sin($phi); + my $eta_new = - $rad * cos($phi); + + my $field = { id => $skycell_id, + xi => $xi_new, + eta => $eta_new, + }; + + push @fields, $field; +} + +### Subdivide list of positions into kd-tree +my $root = { # Root node of tree + contents => \@fields, # Contents of node + position => 'root', # Position in tree + children => {}, # Children of node +}; +my @tasks = ( $root ); +while (scalar @tasks > 1) { + my $node = shift @tasks; + divide_node($node, \@tasks); +} + +### Format tree for magictool +my $mdcTree = print_node($root); # The tree in MDC format +my ($treeFile, $treeName) = tempfile( "magictree.${magic_id}.XXXX", UNLINK => !$save_temps ); +print $treeFile $mdcTree; +close $treeFile; + +### Input tree into database +{ + my $command = "$magictool -inputtree"; + $command .= " -magic_id $magic_id"; + $command .= " -dep_file $treeName"; + $command .= " -dbname $dbname" if defined $dbname; + + # Add the processed file to the database + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magictool -inputtree: $error_code", $magic_id, $error_code); + } + } else { + print "Skipping command: $command\n"; + } +} + +### Pau. + +sub my_die +{ + my $msg = shift; # Warning message on die + my $magic_id = shift; # Magic identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $magic_id and not $no_update) { + my $command = "$magictool -inputtree"; + $command .= " -magic_id $magic_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system($command); + } + exit $exit_code; +} + +# Divide a list into two, returning the lower and upper parts +sub divide_list +{ + my $list = shift; # List to divide + my $index = shift; # Name of index for sorting + + my @sorted = sort { $$a{$index} <=> $$b{$index} } @$list; # Sorted list + my $median = int(scalar @sorted / 2); # Median point of list + my @upper = splice(@sorted, $median); # Upper part of the sorted list + + return (\@sorted, \@upper); +} + +# Create a new node, add it to the parent, and add it to the task list if required +sub new_node +{ + my $parent = shift; # The parent node + my $contents = shift; # Contents of the new node + my $position = shift; # Position description + my $tasks = shift; # Tasks to do + + my $node = { + contents => $contents, + position => $parent->{position} . '_' . $position, + children => {}, + }; + + $parent->{children}->{$position} = $node; + + push @$tasks, $node if scalar @$contents > 4; + + return $node; +} + +# Divide a node +sub divide_node +{ + my $node = shift; # Node to divide + my $tasks = shift; # Tasks to do + + my $position = $node->{position}; + + my $contents = $node->{contents} or die "Can't find contents of node."; # Contents of node + + my ($lower, $upper) = divide_list($contents, 'xi'); + + if (scalar @$lower > 4) { + my ($ll, $lr) = divide_list($lower, 'eta'); + new_node($node, $ll, 'll', $tasks); + new_node($node, $lr, 'lr', $tasks); + } else { + new_node($node, $lower, 'L', $tasks); + } + + if (scalar @$upper > 4) { + my ($ul, $ur) = divide_list($upper, 'eta'); + new_node($node, $ul, 'ul', $tasks); + new_node($node, $ur, 'ur', $tasks); + } else { + new_node($node, $upper, 'U', $tasks); + } + + $node->{contents} = undef; + + return $node; +} + +# Print the contents of a node +sub print_node +{ + my $node = shift; # Node to print + + my $position = $node->{position}; # Position of node + + my $output = "$position\t\tMULTI\n"; # Output text + + if (defined $node->{contents}) { + foreach my $field ( @{$node->{contents}} ) { + my $skycell_id = $field->{id}; # Skycell name + $output .= "$position\t\tSTR\t$skycell_id\n"; + $output .= "$skycell_id\t\tSTR\tNULL\t\# $field->{xi},$field->{eta}\n"; + } + } else { + foreach my $div ( keys %{$node->{children}} ) { + $output .= "$position\t\tSTR\t${position}_$div\n"; + $output .= print_node($node->{children}->{$div}); + } + } + + return $output; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/mdc2list.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/mdc2list.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/mdc2list.pl (revision 22161) @@ -0,0 +1,77 @@ +#!/usr/bin/env perl + +# Program to convert a PS metadata config syntax (usually from one of +# the IPP tools) into a flat tab-delimited list suitable for parsing +# using "split". + +# Copyright (C) 2006 Joshua Hoblitt, Paul A. Price. + +use strict; +use warnings; + +use PS::IPP::Metadata::Config; # Supplies the metadata config parser. + +die "Program to convert a PS metadata config file (usually from one of the IPP\n" . + "tools) into a flat tab-delimited list suitable for parsing using \'split\'.\n\n" . + "Usage: $0 [INFILE [OUTFILE]]\n" if ((join "", @ARGV) =~ /--help/ or scalar @ARGV > 2); + +my $inFile; # Input file +my $outFile; # Output file +if (scalar @ARGV >= 1) { + my $inName = shift @ARGV; + open $inFile, $inName or die "Can't open $inName: $!\n"; + if (scalar @ARGV == 1) { + my $outName = shift @ARGV; + open $outFile, ">", $outName or die "Can't open $outName: $!\n"; + } else { + $outFile = *STDOUT; + } +} else { + $inFile = *STDIN; + $outFile = *STDOUT; +} + +my @input = <$inFile>; # Contents of the metadata config file +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $md = $mdcParser->parse(join "", @input) + or die "unable to parse metadata config doc"; +my $hashes = mds2hashes($md); # An array of hashes +foreach my $pending (@$hashes) { + foreach my $key (keys %$pending) { + print $outFile ( $pending->{$key} . "\t"); + } + print $outFile "\n"; +} + +### Pau. + + +# Given an array of MDs, return an array of hashes +sub mds2hashes +{ + my $mds = shift; # Reference to the metadatas + my @array; # The array of hashes, to be returned + foreach my $md (@$mds) { + my $values = md2hash($md->{value}); + push @array, $values; + } + return \@array; +} + +# Convert the metadata to a hash; in effect, strips out the comment, type and class fields. +sub md2hash +{ + my $values = shift; # Reference to the metadata + my %hash; # Hash, to be returned + foreach my $data (@$values) { + $hash{$data->{name}} = $data->{value}; + } + return \%hash; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/register_exp.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/register_exp.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/register_exp.pl (revision 22161) @@ -0,0 +1,222 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Cache::File; +use Storable qw( freeze thaw ); +use File::Basename qw( basename ); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $ipprc = PS::IPP::Config->new(); + +my ($cache, $exp_id, $exp_tag, $dbname, $verbose, $no_update, $no_op, $logfile); +GetOptions( + 'caches' => \$cache, + 'exp_id|e=s' => \$exp_id, + 'exp_tag|t=s' => \$exp_tag, + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'logfile=s' => \$logfile, +) or pod2usage( 2 ); + +$ipprc->redirect_output($logfile) if $logfile; + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --exp_id --exp_tag", + -exitval => 3) unless + defined $exp_id and + defined $exp_tag; + +# add -detrend UNLESS type is one of SCIENCE listed below (eg, OBJECT) +my @SCIENCE = ( "object", "science" ); # Observation types to NOT mark as detrend +my $DETREND_FLAG = "-end_stage reg"; # Flag to use to mark detrend exposure + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ # register imfile + # label STATISTIC CHIPTOOL FLAG + { name => "exp_name", type => "constant", flag => "-exp_name", dtype => "string" }, # File level + { name => "telescope", type => "constant", flag => "-telescope", dtype => "string" }, # File level + { name => "camera", type => "constant", flag => "-inst", dtype => "string" }, # File level + { name => "filelevel", type => "constant", flag => "-filelevel", dtype => "string" }, # File level + { name => "object", type => "constant", flag => "-object", dtype => "string" }, + { name => "exp_type", type => "constant", flag => "-exp_type", dtype => "string" }, # File level + { name => "filter", type => "constant", flag => "-filter", dtype => "string" }, # File level + { name => "comment", type => "constant", flag => "-comment", dtype => "string" }, # ObsComment + { name => "dateobs", type => "constant", flag => "-dateobs", dtype => "string" }, # File level + { name => "ccd_temp", type => "mean", flag => "-ccd_temp", dtype => "float" }, # CCD temperature + { name => "exp_time", type => "mean", flag => "-exp_time", dtype => "float" }, # Exposure time + { name => "sat_pixel_frac", type => "mean", flag => "-sat_pixel_frac", dtype => "float" }, # Fraction of saturated pixels + { name => "airmass", type => "mean", flag => "-airmass", dtype => "float" }, # Airmass + { name => "ra", type => "mean", flag => "-ra", dtype => "float" }, # Right ascension + { name => "decl", type => "mean", flag => "-decl", dtype => "float" }, # Declination + { name => "posang", type => "mean", flag => "-posang", dtype => "float" }, # Position angle + { name => "alt", type => "mean", flag => "-alt", dtype => "float" }, # Altitude + { name => "az", type => "mean", flag => "-az", dtype => "float" }, # Azimuth + { name => "m1_x", type => "constant", flag => "-m1_x", dtype => "float" }, # M1X + { name => "m1_y", type => "constant", flag => "-m1_y", dtype => "float" }, # M1Y + { name => "m1_z", type => "constant", flag => "-m1_z", dtype => "float" }, # M1Z + { name => "m1_tip", type => "constant", flag => "-m1_tip", dtype => "float" }, # M1TIP + { name => "m1_tilt", type => "constant", flag => "-m1_tilt", dtype => "float" }, # M1TILT + { name => "m2_x", type => "constant", flag => "-m2_x", dtype => "float" }, # M2X + { name => "m2_y", type => "constant", flag => "-m2_y", dtype => "float" }, # M2Y + { name => "m2_z", type => "constant", flag => "-m2_z", dtype => "float" }, # M2Z + { name => "m2_tip", type => "constant", flag => "-m2_tip", dtype => "float" }, # M2TIP + { name => "m2_tilt", type => "constant", flag => "-m2_tilt", dtype => "float" }, # M2TILT + { name => "env_temperature", type => "constant", flag => "-env_temperature", dtype => "float" }, # external temp + { name => "env_humidity", type => "constant", flag => "-env_humidity", dtype => "float" }, # external humidity + { name => "env_wind_speed", type => "constant", flag => "-env_wind_speed", dtype => "float" }, # external wind speed + { name => "env_wind_dir", type => "constant", flag => "-env_wind_dir", dtype => "float" }, # external wind direction + + { name => "-teltemp_m1", type => "constant", flag => "-teltemp_m1", dtype => "float" }, # Primary mirror temps (C) + { name => "-teltemp_m1cell", type => "constant", flag => "-teltemp_m1cell", dtype => "float" }, # Primary mirror support temps (C) + { name => "-teltemp_m2", type => "constant", flag => "-teltemp_m2", dtype => "float" }, # Secondary mirror temps (C + { name => "-teltemp_spider", type => "constant", flag => "-teltemp_spider", dtype => "float" }, # Spider temperatures (C) + { name => "-teltemp_truss", type => "constant", flag => "-teltemp_truss", dtype => "float" }, # Mid truss temperatures (C + { name => "-teltemp_extra", type => "constant", flag => "-teltemp_extra", dtype => "float" }, # Miscellaneous temperatures (C) + + { name => "pon_time", type => "mean", flag => "-pon_time", dtype => "float" }, # time since last power on + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, # background + { name => "bg", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, # Azimuth + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, # Azimuth + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for commands we need +my $missing_tools; +my $regtool = can_run('regtool') + or (warn "can't find regtool" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# setup cache interface +my $c = Cache::File->new( + cache_root => File::Spec->catdir($ENV{'HOME'}, '.pxtools', basename($0)), + default_expires => '7200 sec', +); + +# Get the list of imfiles & their stats +{ + my $command = "$regtool -processedimfile -exp_id $exp_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -processedimfile on exposure id $exp_id: $error_code"); + exit ($error_code); + } + + # Parse the output + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf); + unless ($metadata) { + &my_die ("Unable to parse metadata config doc", $exp_id, $PS_EXIT_PROG_ERROR); + } + + # extract the stats from the metadata + unless ($stats->parse($metadata)) { + &my_die ("Unable to find all values", $exp_id, $PS_EXIT_CONFIG_ERROR); + } +} + +# we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type +if (uc($stats->value_for_flag ("-telescope")) eq "NULL") { &my_die ("telescope not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-inst")) eq "NULL") { &my_die ("inst not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-filelevel")) eq "NULL") { &my_die ("filelevel not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-exp_type")) eq "NULL") { &my_die ("exp_type not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-exp_name")) eq "NULL") { &my_die ("exp_name not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } + +my $command = "$regtool -addprocessedexp"; +$command .= " -exp_id $exp_id"; +$command .= " -exp_tag $exp_tag"; +$command .= " -hostname $host" if defined $host; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +my $exp_type = $stats->value_for_flag ("-exp_type"); + +# Add the detrend flag, if needed +{ + my $object = 0; # Is it an object exposure? + foreach my $scienceType (@SCIENCE) { + if (lc($exp_type) =~ /$scienceType/) { + $object = 1; + last; + } + } + $command .= " $DETREND_FLAG" unless $object; +} + +# Output results to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to run regtool -addprocessedexp for $exp_id: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +### Pau. + +sub cache_run +{ + my %p = @_; + + my $cmd_output = $c->get($p{command}) if $cache; + if (defined $cmd_output) { + return @{thaw $cmd_output}; + } else { + my @output = run(%p); + $c->set($p{command}, freeze \@output) if $cache; + return @output; + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_id = shift; + my $exit_code = shift; + + carp($msg); + if (defined $exp_id and not $no_update) { + my $command = "$regtool -addprocessedexp -exp_id $exp_id -code $exit_code"; + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + system($command); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/register_imfile.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/register_imfile.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/register_imfile.pl (revision 22161) @@ -0,0 +1,244 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Cache::File; +use Storable qw(freeze thaw); +use File::Basename qw( basename); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Config 1.01 qw( :standard ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($cache, $exp_id, $tmp_class_id, $tmp_exp_name, $uri, $dbname, $verbose, $no_update, $no_op, $logfile); +GetOptions( + 'caches' => \$cache, + 'exp_id|e=s' => \$exp_id, + 'tmp_class_id|i=s' => \$tmp_class_id, + 'tmp_exp_name|n=s' => \$tmp_exp_name, + 'uri|u=s' => \$uri, + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'logfile=s' => \$logfile, +) or pod2usage( 2 ); + +$ipprc->redirect_output($logfile) if $logfile; + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --exp_id --tmp_class_id --tmp_exp_name --uri", + -exitval => 3) unless + defined $exp_id and + defined $tmp_class_id and + defined $tmp_exp_name and + defined $uri; + + +my $RECIPE = "REGISTER"; # Recipe to use for ppStats + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "FILE.LEVEL", type => "constant", flag => "-filelevel", dtype => "string" }, # File level + { name => "CLASS.ID", type => "constant", flag => "-class_id", dtype => "string" }, # Real Class ID + { name => "FPA.OBJECT", type => "constant", flag => "-object", dtype => "string" }, # Object + { name => "FPA.OBSTYPE", type => "constant", flag => "-exp_type", dtype => "string" }, # Exposure type + { name => "FPA.FILTER", type => "constant", flag => "-filter", dtype => "string" }, # Filter used + { name => "FPA.COMMENT", type => "constant", flag => "-comment", dtype => "string" }, # Obs Comment + { name => "FPA.AIRMASS", type => "constant", flag => "-airmass", dtype => "float" }, # Airmass + { name => "FPA.RA", type => "constant", flag => "-ra", dtype => "float" }, # Right ascension + { name => "FPA.DEC", type => "constant", flag => "-decl", dtype => "float" }, # Declination + { name => "FPA.ALT", type => "constant", flag => "-alt", dtype => "float" }, # Altitude + { name => "FPA.AZ", type => "constant", flag => "-az", dtype => "float" }, # Azimuth + { name => "FPA.POSANGLE", type => "constant", flag => "-posang", dtype => "float" }, # Position angle + { name => "FPA.TIME", type => "constant", flag => "-dateobs", dtype => "string" }, # Date of observation (UTC) + { name => "FPA.TELESCOPE", type => "constant", flag => "-telescope", dtype => "string" }, # Telescope + { name => "FPA.CAMERA", type => "constant", flag => "-inst", dtype => "string" }, # Instrument + { name => "FPA.M1X", type => "constant", flag => "-m1_x", dtype => "float" }, # M1X + { name => "FPA.M1Y", type => "constant", flag => "-m1_y", dtype => "float" }, # M1Y + { name => "FPA.M1Z", type => "constant", flag => "-m1_z", dtype => "float" }, # M1Z + { name => "FPA.M1TIP", type => "constant", flag => "-m1_tip", dtype => "float" }, # M1TIP + { name => "FPA.M1TILT", type => "constant", flag => "-m1_tilt", dtype => "float" }, # M1TILT + { name => "FPA.M2X", type => "constant", flag => "-m2_x", dtype => "float" }, # M2X + { name => "FPA.M2Y", type => "constant", flag => "-m2_y", dtype => "float" }, # M2Y + { name => "FPA.M2Z", type => "constant", flag => "-m2_z", dtype => "float" }, # M2Z + { name => "FPA.M2TIP", type => "constant", flag => "-m2_tip", dtype => "float" }, # M2TIP + { name => "FPA.M2TILT", type => "constant", flag => "-m2_tilt", dtype => "float" }, # M2TILT + { name => "FPA.ENV.TEMP", type => "constant", flag => "-env_temperature", dtype => "float" }, # external temp + { name => "FPA.ENV.HUMID", type => "constant", flag => "-env_humidity", dtype => "float" }, # external humidity + { name => "FPA.ENV.WIND", type => "constant", flag => "-env_wind_speed", dtype => "float" }, # external wind speed + { name => "FPA.ENV.DIR", type => "constant", flag => "-env_wind_dir", dtype => "float" }, # external wind direction + + { name => "FPA.TELTEMP.M1", type => "constant", flag => "-teltemp_m1", dtype => "float" }, # Primary mirror temps (C) + { name => "FPA.TELTEMP.M1CELL", type => "constant", flag => "-teltemp_m1cell", dtype => "float" }, # Primary mirror support temps (C) + { name => "FPA.TELTEMP.M2", type => "constant", flag => "-teltemp_m2", dtype => "float" }, # Secondary mirror temps (C + { name => "FPA.TELTEMP.SPIDER", type => "constant", flag => "-teltemp_spider", dtype => "float" }, # Spider temperatures (C) + { name => "FPA.TELTEMP.TRUSS", type => "constant", flag => "-teltemp_truss", dtype => "float" }, # Mid truss temperatures (C + { name => "FPA.TELTEMP.EXTRA", type => "constant", flag => "-teltemp_extra", dtype => "float" }, # Miscellaneous temperatures (C) + + { name => "FPA.PON.TIME", type => "constant", flag => "-pon_time", dtype => "float" }, # time since last power on + { name => "CHIP.TEMP", type => "mean", flag => "-ccd_temp", dtype => "float" }, # CCD temperature + { name => "CELL.EXPOSURE", type => "mean", flag => "-exp_time", dtype => "float" }, # Exposure time + { name => "SAT_PIXEL_FRAC", type => "mean", flag => "-sat_pixel_frac", dtype => "float" }, # fraction of saturated pixels + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $regtool = can_run('regtool') + or (warn "Can't find regtool" and $missing_tools = 1); +my $ppStats = can_run('ppStats') + or (warn "Can't find ppStats" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# setup cache interface +# XXX why is this being cached? +my $c = Cache::File->new( + cache_root => File::Spec->catdir($ENV{'HOME'}, '.pxtools', basename($0)), + default_expires => '7200 sec', +); + +my $now_time = localtime(); +printf STDERR "\nstarting ppStats: %s\n", $now_time if $verbose; + +# Run ppStats on the input file +{ + # extract the data from the image header; -level is used to get FILE.LEVEL and CLASS.ID + my $command = "$ppStats $uri -recipe PPSTATS $RECIPE -level"; # Command to run ppStats + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform ppStats on exposure id $exp_id: $error_code", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $error_code); + } + + # Parse the output + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf); # XXX is this join necessary? + unless ($metadata) { + &my_die ("Unable to parse metadata config doc", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_PROG_ERROR); + } + + # extract the stats from the metadata + unless ($stats->parse($metadata)) { + &my_die ("Unable to find all values", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); + } +} + +$now_time = localtime(); +printf STDERR "\ndone with ppStats: %s\n", $now_time if $verbose; + +# we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type +if (uc($stats->value_for_flag ("-telescope")) eq "NULL") { &my_die ("telescope not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-inst")) eq "NULL") { &my_die ("inst not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-filelevel")) eq "NULL") { &my_die ("filelevel not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-class_id")) eq "NULL") { &my_die ("class_id not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-exp_type")) eq "NULL") { &my_die ("exp_type not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } + +my $command = "$regtool -addprocessedimfile"; +$command .= " -exp_id $exp_id"; +$command .= " -exp_name $tmp_exp_name"; # keep the supplied exp_name (could be derived from the file) +$command .= " -tmp_class_id $tmp_class_id"; # the original class_id supplied by the user, replace by ppStats CLASS.ID +$command .= " -uri $uri "; +$command .= " -hostname $host" if defined $host; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +$now_time = localtime(); +printf STDERR "\nrunning regtool update: %s\n", $now_time if $verbose; + +# Push the results into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +$now_time = localtime(); +printf STDERR "\ndone with regtool update: %s\n", $now_time if $verbose; + +sub cache_run +{ + my %p = @_; + + my $cmd_output = $c->get($p{command}) if $cache; + if (defined $cmd_output) { + return @{thaw $cmd_output}; + } else { + my @output = run(%p); + $c->set($p{command}, freeze \@output) if $cache; + return @output; + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_id = shift; + my $exp_name = shift; + my $tmp_class_id = shift; + my $uri = shift; + my $exit_code = shift; + + # for failed imfiles, we insert UNKNOWN for inst, telescope, class_id + + carp($msg); + if (defined $exp_id && defined $tmp_class_id and not $no_update) { + my $command = "$regtool -addprocessedimfile"; + $command .= " -exp_id $exp_id"; + $command .= " -exp_name $exp_name"; + $command .= " -tmp_class_id $tmp_class_id"; + $command .= " -uri $uri "; + $command .= " -telescope UNKNOWN"; + $command .= " -inst UNKNOWN"; + $command .= " -class_id $tmp_class_id"; + $command .= " -code $exit_code"; + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Pau. + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/stack_skycell.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/stack_skycell.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/stack_skycell.pl (revision 22161) @@ -0,0 +1,275 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use File::Temp qw( tempfile ); +use File::Basename; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($stack_id, $dbname, $outroot, $debug, $run_state, $threads, $verbose, $no_update, $no_op, $redirect, $save_temps); +GetOptions( + 'stack_id|d=s' => \$stack_id, # Stack identifier + 'dbname|d=s' => \$dbname, # Database name + 'outroot=s' => \$outroot, # Output root name + 'run-state=s' => \$run_state, + 'debug' => \$debug, # Print to stdout + 'threads=s' => \$threads, # Number of threads to use for ppStack + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --stack_id --outroot --run-state", + -exitval => 3, + ) unless defined $stack_id + and defined $outroot + and defined $run_state; + +# XXX camera is not known here; cannot use filerules... +# my $logDest = $ipprc->filename("LOG.EXP", $outroot); + +my $logDest = "$outroot.log"; +$ipprc->redirect_output($logDest) if $redirect; + +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC STACKTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, +# { name => "DT_STACK", type => "sum", flag => "-dtime_stack", dtype => "float" }, + { name => "GOOD_PIXEL_FRAC", type => "mean", flag => "-good_frac", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); +my $ppStack = can_run('ppStack') or (warn "Can't find ppStack" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of components for stacking +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $files; +{ + my $command = "$stacktool -inputskyfile -stack_id $stack_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -inputskyfile: $error_code", $stack_id, $error_code); + } + + if (@$stdout_buf == 0) { + &my_die("No input skyfiles selected", $stack_id, $PS_EXIT_PROG_ERROR); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $stack_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $stack_id, $PS_EXIT_PROG_ERROR); +} + +&my_die("Stack list contains less than two elements", $stack_id, $PS_EXIT_SYS_ERROR) unless + scalar @$files >= 2; + +# Parse the list of input files to get the tesselation, skycell identifiers and camera +my $skycell_id; # Skycell identifier +my $tess_id; # Tesselation identifier +my $camera; # Camera +foreach my $file (@$files) { + # skip warps which are specified as 'ignored' + if ($file->{ignored}) { next; } + if (defined $tess_id) { + &my_die("Tesselation identifiers don't match", $stack_id, $PS_EXIT_SYS_ERROR) unless + $file->{tess_id} eq $tess_id; + } else { + $tess_id = $file->{tess_id}; + } + if (defined $skycell_id) { + &my_die("Skycell identifiers don't match", $stack_id, $PS_EXIT_SYS_ERROR) unless + $file->{skycell_id} eq $skycell_id; + } else { + $skycell_id = $file->{skycell_id}; + } + if (defined $camera) { + &my_die("Cameras don't match", $stack_id, $PS_EXIT_SYS_ERROR) unless $file->{camera} eq $camera; + } else { + $camera = $file->{camera}; + } +} + +&my_die("Can't find camera", $stack_id, $PS_EXIT_SYS_ERROR) unless defined $camera; +$ipprc->define_camera($camera); + +# Generate MDC file with the inputs +my $tess_base = basename($tess_id); +my ($listFile, $listName) = tempfile("/tmp/$tess_base.$skycell_id.stk.$stack_id.list.XXXX", UNLINK => !$save_temps ); +my $num = 0; +my $inputSources; # Sources to use as stamps +foreach my $file (@$files) { + if ($file->{ignored}) { next; } + + print $listFile "INPUT$num\tMETADATA\n"; + $num++; + + my $image = $file->{uri}; # Image name + my $mask = $ipprc->filename( "PSWARP.OUTPUT.MASK", $file->{path_base} ); # Mask name + my $weight = $ipprc->filename( "PSWARP.OUTPUT.WEIGHT", $file->{path_base} ); # Weight name + my $psf = $ipprc->filename( "PSPHOT.PSF.SKY.SAVE", $file->{path_base} ); # PSF name + my $sources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $file->{path_base}); # Sources name + + &my_die("Image $image does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $image ); + &my_die("Mask $mask does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $mask ); + &my_die("Weight $weight does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $weight ); + &my_die("PSF $psf does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $psf ); + &my_die("Sources $sources does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $sources ); + + print $listFile "\tIMAGE\tSTR\t" . $image . "\n"; + print $listFile "\tMASK\tSTR\t" . $mask . "\n"; + print $listFile "\tWEIGHT\tSTR\t" . $weight . "\n"; + print $listFile "\tPSF\tSTR\t" . $psf . "\n"; + print $listFile "\tSOURCES\tSTR\t" . $sources . "\n"; + + ### XXX NEED TO UPDATE THESE appropriately + print $listFile "\tWEIGHTING\tF32\t" . 1.0 . "\n"; + + print $listFile "END\n\n"; +} + +# Get the output filenames +my $outputName = $ipprc->filename("PPSTACK.OUTPUT", $outroot); +my $outputMask = $ipprc->filename("PPSTACK.OUTPUT.MASK", $outroot); +my $outputWeight = $ipprc->filename("PPSTACK.OUTPUT.WEIGHT", $outroot); +my $outputSources = $ipprc->filename("PSPHOT.OUT.CMF.MEF", $outroot); ## this must be consistent with the value in diff_skycell.pl:101 +## use an explicit stack name for psphot output objects +#my $bin1Name = $ipprc->filename("PPSTACK.BIN1", $outroot); +#my $bin2Name = $ipprc->filename("PPSTACK.BIN2", $outroot); +my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); +my $configuration = $ipprc->filename("PPSTACK.CONFIG", $outroot); + +# Perform stacking +unless ($no_op) { + my $command = "$ppStack $listName $outroot"; + $command .= " -stats $outputStats"; + $command .= " -recipe PPSUB STACK"; + $command .= " -recipe PPSTATS WARPSTATS"; + $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE"; + $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; + $command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF"; + $command .= " -photometry"; + $command .= " -threads $threads" if defined $threads; + $command .= " -debug-stack" if defined $debug; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dumpconfig $configuration"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStack: $error_code", $stack_id, $error_code); + } + &my_die("Couldn't find expected output file: $outputName", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); + &my_die("Couldn't find expected output file: $outputMask", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + &my_die("Couldn't find expected output file: $outputWeight", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); + &my_die("Couldn't find expected output file: $outputSources", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); +# &my_die("Couldn't find expected output file: $bin1Name", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin1Name); +# &my_die("Couldn't find expected output file: $bin2Name", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin2Name); + &my_die("Couldn't find expected output file: $outputStats", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); + + # Get the statistics on the stacked image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $stack_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + my $metadata = $mdcParser->parse(join "", @contents) or + &my_die("Unable to parse metadata config doc", $stack_id, $PS_EXIT_PROG_ERROR); + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $stack_id, $PS_EXIT_PROG_ERROR); +} + +unless ($no_update) { + + # Add the stack result + { + my $command = "$stacktool -addsumskyfile -stack_id $stack_id -uri $outputName -path_base $outroot"; + $command .= $stats->cmdflags(); + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -addsumskyfile: $error_code", $stack_id, $error_code); + } + } + + # Register the run as completed + # XXX why is this needed? the stackRun is set to 'full' by the stacktool -addsumskyfile command... + if (0) { + my $command = "$stacktool -updaterun -stack_id $stack_id -state full"; # Command to run stacktool + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform stacktool -updaterun: $error_code\n"); + exit($error_code); + } + } +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $stack_id = shift; # Stack identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $stack_id and not $no_update) { + my $command = "$stacktool -addsumskyfile -stack_id $stack_id -code $exit_code"; + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/summit_copy.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/summit_copy.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/summit_copy.pl (revision 22161) @@ -0,0 +1,163 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Config 1.01 qw( :standard ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use Sys::Hostname; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +## report the program and machine +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +# Parse the command-line arguments +my ( $uri, $filename, $compress, $bytes, $md5, $nebulous, $exp_name, $inst, $telescope, $class, $class_id, $end_stage, $workdir, + $dbname, $verbose, $no_update, $no_op, $timeout ); +GetOptions( + 'uri=s' => \$uri, # source location of file on data store + 'filename=s' => \$filename, # target location of file on local system + 'compress' => \$compress, # request file in compressed format + 'bytes=s' => \$bytes, # reported file size in bytes + 'md5=s' => \$md5, # reported md5 checksum + 'nebulous' => \$nebulous, # use nebulous for the target file + 'exp_name=s' => \$exp_name, # Exposure name + 'inst=s' => \$inst, # Instrument + 'telescope=s' => \$telescope, # Telescope + 'class=s' => \$class, # Class level + 'class_id=s' => \$class_id, # Class identifier + 'end_stage=s' => \$end_stage, # end of processing + 'workdir=s' => \$workdir, # workdir + 'dbname=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'timeout=s' => \$timeout, # passed through to dsget +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --uri --filename --exp_name --inst --telescope --class --class_id --workdir", + -exitval => 3) + unless defined $uri + and defined $filename + and defined $exp_name + and defined $inst + and defined $telescope + and defined $class + and defined $class_id + and defined $workdir; + +# Look for programs we need +my $missing_tools; +my $dsget = can_run('dsget') + or (warn "Can't find dsget" and $missing_tools = 1); +my $pztool = can_run('pztool') + or (warn "Can't find pztool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# dsget command +my $command; +$command = "$dsget --uri $uri --filename $filename"; +$command .= " --compress" if defined $compress; +$command .= " --bytes $bytes" if defined $bytes; +$command .= " --nebulous" if defined $nebulous; +$command .= " --md5 $md5" if defined $md5; +$command .= " --timeout $timeout" if defined $timeout; + +# run command +unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) + = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + my_die("Unable to perform dsget: $error_code", + $exp_name, + $inst, + $telescope, + $class, + $class_id, + $uri, + $error_code + ); + } +} else { + print "skipping command: $command\n"; +} + +# command to update database +$command = "$pztool -copydone"; +$command .= " -exp_name $exp_name"; +$command .= " -inst $inst"; +$command .= " -telescope $telescope"; +$command .= " -class $class"; +$command .= " -class_id $class_id"; +$command .= " -uri $filename"; +$command .= " -workdir $workdir"; +$command .= " -end_stage $end_stage" if defined $end_stage; +$command .= " -dbname $dbname" if defined $dbname; + +# update the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) + = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform $command: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_name = shift; # Chiptool identifier + my $inst = shift; # Chiptool identifier + my $telescope = shift; # Class identifier + my $class = shift; # Class identifier + my $class_id = shift; # Class identifier + my $uri = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + warn $msg; + unless ($no_update) { + # command to update database + my $command; + $command = "$pztool -copydone"; + $command .= " -exp_name $exp_name"; + $command .= " -inst $inst"; + $command .= " -telescope $telescope"; + $command .= " -class $class"; + $command .= " -class_id $class_id"; + $command .= " -uri $uri"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + + system ($command); + } + + exit $exit_code; +} + +# XXX: I don't think that we need this - JH +#END { +# my $status = $?; +# system("sync") == 0 +# or die "failed to execute sync: $!" ; +# $? = $status; +#} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/warp_overlap.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/warp_overlap.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/warp_overlap.pl (revision 22161) @@ -0,0 +1,314 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use File::Temp qw( tempfile ); +use File::Spec; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($warp_id, $camera, $tess_dir, $dbname, $verbose, $no_update, $no_op, $logfile, $save_temps); +GetOptions( + 'warp_id|i=s' => \$warp_id, # Warp identifier + 'camera|c=s' => \$camera, # Camera name + 'tess_dir=s' => \$tess_dir, # Tessellation directory + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations + 'logfile=s' => \$logfile, + 'save-temps' => \$save_temps, # Save temporary files? +) or pod2usage( 2 ); + +$ipprc->redirect_output($logfile) if $logfile; + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --warp_id --camera --tess_dir", + -exitval => 3, +) unless defined $warp_id + and defined $camera + and defined $tess_dir; + +$ipprc->define_camera($camera); + +# Look for programs we need +my $missing_tools; +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $dvoImageOverlaps = can_run('dvoImageOverlaps') or (warn "Can't find dvoImageOverlaps" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get list of component imfiles for exposure +my $imfiles; +{ + my $command = "$warptool -imfile -warp_id $warp_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -imfile: $error_code", $warp_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_id, $PS_EXIT_PROG_ERROR); + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $warp_id, $PS_EXIT_PROG_ERROR); +} + +# Where do we get the astrometry source from? +my $astromSource; # The astrometry source filerule (eg, PSASTRO.OUTPUT, PSASTRO.OUTPUT.MEF) +my $astromAccept; # Accept the astrometry unconditionally? +{ + my $command = "$ppConfigDump -camera $camera -dump-recipe PSWARP -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_id, $PS_EXIT_PROG_ERROR); + $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE'); + $astromAccept = metadataLookupBool($metadata, 'ASTROM.ACCEPT'); +} + +# Determine the imfile/skycell overlaps +my @overlaps = (); +unless ($no_op) { + # Calculate the overlaps between imfiles and skycells + + # tess_dir is the DVO db holding the tessalations + # this may be an abstract name in site.config:TESSELLATIONS, a URI, or an absolute path. + # convert this to an absolute path + my $tess_dir_abs = $ipprc->tessellation_catdir( $tess_dir ); # Tessellation catdir for DVO + $tess_dir_abs = $ipprc->convert_filename_absolute( $tess_dir_abs ); + + my %unique_skycells = (); # Identified skycells (all unique by virtue of hash property) + + # astrometry is always determined at the camera stage; we have a MEF astrometry file from psastro + my $imfile = $imfiles->[0]; + my $camRoot = $imfile->{cam_path_base}; + my $astromFile = $ipprc->filename($astromSource, $camRoot); # Astrometry file + if (!$astromFile) { + &my_die("Unable to determine the astrometry source", $warp_id, $PS_EXIT_DATA_ERROR); + } + $astromFile = $ipprc->file_resolve($astromFile); + if (!$astromFile) { + &my_die("Unable to resolve real astrometry source filename", $warp_id, $PS_EXIT_DATA_ERROR); + } + + my @matchlist = get_overlaps($astromFile, $tess_dir_abs, $astromAccept); # List of overlaps + if (! @matchlist) { + &my_die("Unable to perform dvoImageOverlaps: missing astrometry", $warp_id, $PS_EXIT_DATA_ERROR); + } + # Match each of the imfiles to this list (the input images may be split, but the astrometry is not) + # tess_dir (not tess_dir_abs) is supplied here so it may be written to the db + foreach my $imfile (@$imfiles) { + extract_overlaps(\@matchlist, $imfile, $astromFile, $tess_dir, \@overlaps, \%unique_skycells); + } +} else { + # create an overlap with an entry for each skycell:imfile match + foreach my $imfile (@$imfiles) { + my %overlap = (); + $overlap{skycell_id} = 'default'; + $overlap{tess_dir} = 'default'; + $overlap{cam_id} = $imfile->{cam_id}; + $overlap{class_id} = $imfile->{class_id}; + $overlap{fault} = $imfile->{fault}; + push @overlaps, \%overlap; + } +} + +# If no overlaps are found, we +# keep running this step over and over (a successful warptool -addoverlap prevents +# successive warptime -imfile from running. +&my_die("Unable to find any overlaps", $warp_id, $PS_EXIT_PROG_ERROR) if scalar @overlaps == 0; + +# Generate a MDC file with the overlaps +my ($overlapFile, $overlapName) = tempfile( "/tmp/overlaps.wrp.$warp_id.mdc.XXXX", UNLINK => !$save_temps ); +print $overlapFile "warpSkyCellMap MULTI\n\n"; +foreach my $overlap (@overlaps) { + print $overlapFile "warpSkyCellMap METADATA\n"; + print $overlapFile " warp_id S32 $warp_id\n"; + print $overlapFile " skycell_id STR $overlap->{skycell_id}\n"; + # XXX convert tess_id here to tess_dir when db scheme is updated + print $overlapFile " tess_id STR $overlap->{tess_dir}\n"; + print $overlapFile " cam_id S32 $overlap->{cam_id}\n"; + print $overlapFile " class_id STR $overlap->{class_id}\n"; + print $overlapFile " fault S16 $overlap->{fault}\n"; + print $overlapFile "END\n\n"; +} +close $overlapFile; + +system "cat $overlapName" if $verbose; + +# Add the processed file to the database +unless ($no_update) { + my $command = "$warptool -addoverlap -mapfile $overlapName"; # Command to run warptool + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform warptool -addoverlap: $error_code\n"); + exit($error_code); + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $warp_id = shift; # Warp identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $warp_id and not $no_update) { + my $command = "$warptool -addoverlap"; + $command .= " -warp_id $warp_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Run dvoImageOverlaps to get the overlaps; return the output +sub get_overlaps +{ + my $filename = shift; # Filename on which to run dvoImageOverlaps + my $tess_dir_abs = shift; # Tessellation directory + my $accept = shift; # Do we use the -accept-astrom flag? + + my $command = "$dvoImageOverlaps -D CATDIR $tess_dir_abs " . $filename; + $command .= ' -accept-astrom' if $accept; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + if (!$success) { + print "Unable to calculate overlaps from $filename\n"; + print "STDOUT:\n"; + foreach my $line (@$stdout_buf) { + print ">>>$line"; + } + print "STDERR:\n"; + foreach my $line (@$stderr_buf) { + print ">>>$line"; + } + return 0; + } + return split ('\n', (join "", @$stdout_buf)); +} + +# Extract a list of overlaps for an imfile +# +# The command "dvoImageOverlaps FILENAME" returns: +# FILENAME[HDU] : SKYCELL.ID +# eg: 729534pa.cmf[ccd00.hdr] : skycell.051.fits +# for the case of split data, we need to identify the input imfiles based on this information +sub extract_overlaps +{ + my $matches = shift; # Reference to list of skycells from dvoImageOverlaps + my $imfile = shift; # Imfile information + my $filename = shift; # Filename used with dvoImageOverlaps + my $tess_dir = shift; # Tessellation identifier + my $overlaps = shift; # Reference to list of overlaps + my $unique_skycells = shift; # Reference to hash of found skycells + + # Get rid of the path + my @dirlist = File::Spec->splitdir( $filename ); # The elements of the full path + $filename = pop @dirlist; + + # Work out how to identify this imfile in the output + my $fileLevel = $imfile->{filelevel}; + my $entry; # How to identify this imfile in the dvoImageOverlaps output + + if (lc($fileLevel) eq "chip") { + # in the case of SPLIT images, all CLASSes are included in the output list + # we need to pull out the single CLASS_ID for this imfile from the full list + my $class_id = $imfile->{class_id}; + my $chipRoot = $imfile->{chip_path_base}; + my $extname = $ipprc->extname_rule("CMF.HEAD", $class_id); # MEF psastro output + + $entry = $filename . '\[' . $extname . '\]'; + print STDERR "entry: $entry, class: $class_id, extname: $extname, chiproot: $chipRoot\n" if $verbose; + } else { + # in the case of MEF or SINGLE images, there is only a single CLASS in the output list + $entry = $filename; + print STDERR "entry: $entry\n" if $verbose; + } + + my @skycells = &select_skycells($entry, @$matches); # Matching skycells + my $Nskycells = @skycells; + printf STDERR "Nskycells: $Nskycells\n" if $verbose; + foreach my $skycell (@skycells) { + my %overlap = (); # Overlap information for warptool + $overlap{skycell_id} = $skycell; + $overlap{tess_dir} = $tess_dir; + $overlap{cam_id} = $imfile->{cam_id}; + $overlap{class_id} = $imfile->{class_id}; + $overlap{fault} = $imfile->{fault}; + push @$overlaps, \%overlap; + + printf STDERR "overlap: %s : %s , %s\n", $skycell, $imfile->{cam_id}, $imfile->{class_id} if $verbose; + + $unique_skycells->{$skycell} = 1; + } + + return; +} + +# Find skycells in the list that come from a particular entry +sub select_skycells +{ + my $entry = shift; # File+Ext to search for + my @list = @_; # List of "File+Ext : skycell" + + my @skycells = (); + my %unique = (); # Ensure we only return unique skycells for this entry + + foreach my $line (@list) { + if ($line =~ m|$entry|) { + my ($skycell) = $line =~ m|$entry\S*\s+:\s+(\S+)|; + if (not defined $unique{$skycell}) { + push @skycells, $skycell; + $unique{$skycell} = 1; + } + } + } + return @skycells; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/warp_skycell.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/warp_skycell.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/scripts/warp_skycell.pl (revision 22161) @@ -0,0 +1,321 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use File::Temp qw( tempfile ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($warp_id, $skycell_id, $tess_dir, $camera, $dbname, $outroot, $threads, $run_state, $verbose, $no_update, $no_op, $redirect, $save_temps); +GetOptions( + 'warp_id|i=s' => \$warp_id, # Warp identifier + 'skycell_id|s=s' => \$skycell_id, # Skycell identifier + 'tess_dir|s=s' => \$tess_dir, # Tesselation identifier + 'camera|c=s' => \$camera, # Camera name + 'dbname|d=s' => \$dbname, # Database name + 'outroot=s' => \$outroot, # Output root name + 'threads=s' => \$threads, # Number of threads to use for pswarp + 'run-state=s' => \$run_state, # 'new' or 'update' + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --warp_id --skycell_id --tess_dir --camera --outroot --run-state", + -exitval => 3, +) unless defined $warp_id + and defined $skycell_id + and defined $tess_dir + and defined $camera + and defined $outroot + and defined $run_state; + +$ipprc->define_camera($camera); + +my $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id); +$logDest .= ".update" if ($run_state eq 'update'); + +$ipprc->redirect_output($logDest) if $redirect; + +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC WARPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "DT_WARP", type => "sum", flag => "-dtime_warp", dtype => "float" }, + { name => "GOOD_PIXEL_FRAC", type => "mean", flag => "-good_frac", dtype => "float" }, + { name => "RANGE.XMIN", type => "mean", flag => "-xmin", dtype => "int" }, + { name => "RANGE.XMAX", type => "mean", flag => "-xmax", dtype => "int" }, + { name => "RANGE.YMIN", type => "mean", flag => "-ymin", dtype => "int" }, + { name => "RANGE.YMAX", type => "mean", flag => "-ymax", dtype => "int" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $pswarp = can_run('pswarp') or (warn "Can't find pswarp" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component imfiles for exposure +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $imfiles; +{ + # XXX change -tess_id to -tess_dir when db schema is updated + my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_dir"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -scmap: $error_code", $warp_id, $skycell_id, $tess_dir, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR); + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR); +} + +# Where do we get the astrometry source from? +my $astromSource; # The astrometry source +{ + my $command = "$ppConfigDump -camera $camera -dump-recipe PSWARP -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_id, $PS_EXIT_PROG_ERROR); + $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE'); +} + +my $outputImage = $ipprc->filename("PSWARP.OUTPUT", $outroot, $skycell_id ); +my $outputMask = $ipprc->filename("PSWARP.OUTPUT.MASK", $outroot, $skycell_id); +my $outputWeight = $ipprc->filename("PSWARP.OUTPUT.WEIGHT", $outroot, $skycell_id); +my $outputSources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $outroot, $skycell_id); +my $outputPSF = $ipprc->filename("PSPHOT.PSF.SKY.SAVE", $outroot); +my $outputBin1 = $ipprc->filename("PSWARP.BIN1", $outroot, $skycell_id ); +my $outputBin2 = $ipprc->filename("PSWARP.BIN2", $outroot, $skycell_id ); +my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot, $skycell_id ); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot, $skycell_id); +my $configuration = $ipprc->filename("PSWARP.CONFIG", $outroot, $skycell_id); + +if ($run_state eq 'update') { + $traceDest .= ".update"; + $outputStats .= ".update"; +} + +my $skyFile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id ); +$ipprc->skycell_file( $tess_dir, $skycell_id, $skyFile, $verbose ) or &my_die("Unable to generate template skycell", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR); +## XXX this seems to have insufficient error checking: dvoImageExtract can fail to write and still return a valid exit status + +# Get list of filenames +my $tempOutRoot = "/tmp/warp.$camera.$warp_id.$skycell_id"; +my ($imageFile, $imageName) = tempfile( "$tempOutRoot.image.list.XXXX", UNLINK => !$save_temps); +my ($maskFile, $maskName) = tempfile( "$tempOutRoot.mask.list.XXXX", UNLINK => !$save_temps); +my ($weightFile, $weightName) = tempfile( "$tempOutRoot.weight.list.XXXX", UNLINK => !$save_temps); +my ($astromFile, $astromName) = tempfile( "$tempOutRoot.astrom.list.XXXX", UNLINK => !$save_temps); + +my $wrote_astrom = 0; +foreach my $imfile (@$imfiles) { + my $image = $imfile->{uri}; # Image name + my $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name + my $weight = $ipprc->filename("PPIMAGE.CHIP.WEIGHT", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name + + &my_die("Couldn't find input file: $image", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($image); + &my_die("Couldn't find input file: $mask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($mask); + + # Astrometry file: astrometry is done at the camera stage, and always results in a MEF file + # XXX allow an option to use the image header astrometry? + my $astrom = $ipprc->filename($astromSource, $imfile->{cam_path_base}); + + &my_die("Couldn't find input file: $astrom", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless defined $astrom and $ipprc->file_exists($astrom); + + print $imageFile "$image\n"; + print $maskFile "$mask\n"; + print $weightFile "$weight\n"; + + if (!$wrote_astrom) { + print $astromFile "$astrom\n"; + $wrote_astrom = 1; + } +} +close $imageFile; +close $maskFile; +close $weightFile; +close $astromFile; + + +# Run pswarp +my $accept = 1; # Accept the skycell? +my $do_stats; +unless ($no_op) { + my $command = "$pswarp"; + $command .= " -list $imageName"; + $command .= " -masklist $maskName"; + $command .= " -weightlist $weightName"; + $command .= " -astromlist $astromName"; + $command .= " $outroot $skyFile"; + $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE"; + $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; + $command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF"; + $command .= " -psf"; # Turn on PSF determination + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -threads $threads" if defined $threads; + $command .= " -dbname $dbname" if defined $dbname; + if ($run_state eq 'new') { + $command .= " -dumpconfig $configuration"; + $do_stats = 1; + } else { + $command .= " -ipprc $configuration"; + } + if ($do_stats) { + $command .= " -recipe PPSTATS WARPSTATS"; + $command .= " -stats $outputStats"; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform pswarp: $error_code", $warp_id, $skycell_id, $tess_dir, $error_code); + } + + if ($do_stats) { + # Check first for the stats file, and if the ACCEPT flag is set. + &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); + # Get the statistics on the warped image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or die "Can't open statistics file $outputStats: $!\n"; + my @contents = <$statsFile>; # Contents of file + close $statsFile; + my $contents = join "", @contents; + + my $metadata = $mdcParser->parse($contents) + or &my_die("Unable to parse metadata config", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR); + $accept = metadataLookupBool($metadata, "ACCEPT"); + + # $accept is set above based on the fraction of lit pixels + # XXX for some files, there may not be enough stars to find a good psf. these should be dropped as well + if ($accept && !$ipprc->file_exists($outputPSF)) { + $accept = 0; + } + + if ($accept) { + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR); + + &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage); + &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); + &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); + # &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputPSF); + # &my_die("Couldn't find expected output file: $outputBin1", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1); + # &my_die("Couldn't find expected output file: $outputBin2", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2); + } + + unless ($no_update) { + # XXX change -tess_id to -tess_dir when db is updated + my $command = "$warptool -addwarped"; + $command .= " -warp_id $warp_id"; + $command .= " -skycell_id $skycell_id"; + $command .= " -tess_id $tess_dir"; + $command .= " -path_base $outroot"; # needed for logfile lookups + $command .= " -ignore" if not $accept; # Completed succesfully, but can't produce product + $command .= " -uri $outputImage" if $accept; + $command .= $stats->cmdflags() if $accept; + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform warptool -addwarped: $error_code\n"); + exit($error_code); + } + } + } else { + # $run_state eq 'update' + unless ($no_update) { + my $command = "$warptool -tofullskyfile"; + $command .= " -warp_id $warp_id"; + $command .= " -skycell_id $skycell_id"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform warptool -addwarped: $error_code\n"); + exit($error_code); + } + } + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $warp_id = shift; # Warp identifier + my $skycell_id = shift; # Skycell identifier + my $tess_dir = shift; # Tesselation identifier + my $exit_code = shift; # Exit code to add + + warn($msg); + if (defined $warp_id and defined $skycell_id and defined $tess_dir and not $no_update) { + # XXX change -tess_id to -tess_dir when db is updated + my $command = "$warptool"; + if ($run_state eq 'new') { + $command .= " -addwarped"; + $command .= " -tess_id $tess_dir"; + $command .= " -path_base $outroot"; + $command .= " -hostname $host" if defined $host; + } else { + $command .= " -updateskyfile"; + } + $command .= " -warp_id $warp_id"; + $command .= " -skycell_id $skycell_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + run(command => $command, verbose => $verbose); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080918/ippScripts/t/00_distribution.t =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippScripts/t/00_distribution.t (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippScripts/t/00_distribution.t (revision 22161) @@ -0,0 +1,25 @@ +#!/usr/bin/env perl + +# Copyright (C) 2005 Joshua Hoblitt +# +# $Id: 00_distribution.t,v 1.1 2007-01-30 01:52:16 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use lib qw( ./lib ./t ); + +use Test::More; + +# example taken from Test::Distribution Pod + +BEGIN { + eval { + require Test::Distribution; + }; + if($@) { + plan skip_all => 'Test::Distribution not installed'; + } else { + import Test::Distribution; # not => qw( podcover ); + } +} Index: /branches/eam_branches/eam_branch_20080918/ippTasks/.cvsignore =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/.cvsignore (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/.cvsignore (revision 22161) @@ -0,0 +1,9 @@ +Makefile +Makefile.in +aclocal.m4 +autom4te.cache +config.log +config.status +configure +install-sh +missing Index: /branches/eam_branches/eam_branch_20080918/ippTasks/Makefile.am =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/Makefile.am (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/Makefile.am (revision 22161) @@ -0,0 +1,44 @@ +task_files = \ + detrend.mkruns.pro \ + detrend.norm.pro \ + detrend.process.pro \ + detrend.reject.pro \ + detrend.resid.pro \ + detrend.stack.pro \ + automate.pro \ + pantasks.pro \ + register.pro \ + chip.pro \ + camera.pro \ + fake.pro \ + warp.pro \ + magic.pro \ + diff.pro \ + stack.pro \ + summit.copy.pro \ + replicate.pro \ + pstamp.pro + +other_files = \ + ipphosts.manoa.config \ + ipphosts.mhpcc.config \ + site.manoa.pro \ + site.mhpcc.pro \ + simtest.pro \ + simtest.basic.config \ + simtest.basic.auto \ + simtest.detverify.config \ + simtest.detverify.auto \ + simtest.flatcorr.config \ + simtest.flatcorr.auto + +pantasksdir = $(datadir)/pantasks +ipptasksdir = $(pantasksdir)/modules +ipptasks_DATA = $(task_files) $(other_files) + +install-data-hook: + chmod 0755 $(panstasks) $(ipptasksdir) + +EXTRA_DIST = $(task_files) $(other_files) + +ACLOCAL_AMFLAGS = -I m4 Index: /branches/eam_branches/eam_branch_20080918/ippTasks/autogen.sh =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/autogen.sh (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/autogen.sh (revision 22161) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=ippTasks +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +#($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $LIBTOOLIZE installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" +# DIE=1 +#} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +#($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $AUTOHEADER installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" +# DIE=1 +#} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +#$LIBTOOLIZE --copy --force || echo "$LIBTOOLIZE failed" +$ACLOCAL || echo "$ACLOCAL failed" +#$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: /branches/eam_branches/eam_branch_20080918/ippTasks/automate.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/automate.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/automate.pro (revision 22161) @@ -0,0 +1,456 @@ +## automate.pro : autorun the detrend analysis : -*- sh -*- + +# example automation stage +# automate METADATA +# name STR DARK +# check STR "detselect -search -inst SIMTEST -det_type BIAS -dbname eamtest" +# ncheck S32 1 +# launch STR "dettool -definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type DARK -select_exp_type DARK -dbname eamtest" +# block STR "dettool -runs -active -det_type DARK -dbname eamtest" +# END + +# an entry in the 'automate' book goes through the following states: +# INIT (book loaded from input file) +# CHECK (stage prereqs being checked) +# READY (prereqs satisfield, ready to be launched) +# LAUNCH (action being performed) +# DONE (action finished) + +macro automate.reset + ## probably should not always init + book init automate +end + +macro automate.load + if ($0 != 4) + echo "USAGE: automate.load (filename) (camera) (dbname)" + break + end + queueload tmp -x "cat $MODULES:0/$1" + + pwd -var cwd + + ## interpolate standard values + queuesubstr tmp @CAMERA@ $2 + queuesubstr tmp @DBNAME@ $3 + queuesubstr tmp @CWD@ $cwd + + ipptool2book tmp automate -key name -uniq -setword pantaskState INIT.BLOCK + queuedelete tmp + + ## Pull out the ones that are to be run regularly + local npages + book npages automate -var npages + for i 0 $npages + book getpage automate $i -var pageName + if ("$pageName" != "NULL") + book getword automate $pageName regular -var regularCommand + if ("$regularCommand" != "NULL") + book setword automate $pageName pantaskState INIT.REGULAR + end + end + end + +end + +macro automate.on + task automate.block + active true + end + task automate.check + active true + end + task automate.launch + active true + end + task automate.regular + active true + end +end + +macro automate.off + task automate.block + active false + end + task automate.check + active false + end + task automate.launch + active false + end + task automate.regular + active false + end +end + +$automate_Nblock = -1 +task automate.block + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + task.exec + local Npage pageName + + # how many pages are waiting to be started? + book npages automate -var Npage -key pantaskState INIT.BLOCK + if ($Npage == 0) + if ($VERBOSE >= 2) + echo "no entries in INIT.BLOCK state" + end + break + end + + # cycle over the number of INIT.BLOCK pages: no point to loop over the others + $automate_Nblock ++ + if ($automate_Nblock >= $Npage) set automate_Nblock = 0 + + # search the automate book for an entry which is unstarted (state INIT.BLOCK) + book getpage automate $automate_Nblock -var pageName -key pantaskState INIT.BLOCK + if ("$pageName" == "NULL") + if ($VERBOSE >= 2) + echo "entry $automate_Nblock not in INIT.BLOCK state" + end + break + end + + book getword automate $pageName block -var blockCommand + if (("$blockCommand" == "NULL") || ("$blockCommand" == "NONE")) + # if there is no block needed, we can immediate progress to the next stage (INIT.CHECK) + book setword automate $pageName pantaskState INIT.CHECK + if ($VERBOSE >= 2) + echo "$pageName is ready : INIT.CHECK" + end + break + end + + book setword automate $pageName pantaskState RUN.BLOCK + + if ($VERBOSE >= 2) + echo "starting automate block for $pageName" + echo "command $blockCommand" + end + + options $pageName + command $blockCommand + end + + task.exit $EXIT_SUCCESS + local pageName Npage + + $pageName = $options:0 + + # convert 'stdout' to book format + # XXX to use other tests, we'll need to modify this + ipptool2book stdout tmpBlock + if ($VERBOSE > 2) + book listbook tmpBlock + end + + # if the block test returns any valid pages (valid results), the block is set, don't move to check + book npages tmpBlock -var Npage + if ($Npage == 0) + if ($VERBOSE >= 2) + echo "$pageName is not blocked, ready for CHECK" + end + book setword automate $pageName pantaskState INIT.CHECK + else + if ($VERBOSE >= 2) + echo "$pageName is blocked, not ready for CHECK" + end + book setword automate $pageName pantaskState INIT.BLOCK + end + + # drop the detExt book after we've grabbed the state + book delete tmpBlock + end + + # all failures here (what state?) + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +$automate_Ncheck = -1 +task automate.check + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + task.exec + local Npage pageName + + # how many pages are waiting to be started? + book npages automate -var Npage -key pantaskState INIT.CHECK + if ($Npage == 0) + if ($VERBOSE >= 2) + echo "no entries in INIT.CHECK state" + end + break + end + + # cycle over the number of INIT.CHECK pages: no point to loop over the others + $automate_Ncheck ++ + if ($automate_Ncheck >= $Npage) set automate_Ncheck = 0 + + # search the automate book for an entry which is unstarted (state INIT.CHECK) + book getpage automate $automate_Ncheck -var pageName -key pantaskState INIT.CHECK + if ("$pageName" == "NULL") + if ($VERBOSE >= 2) + echo "entry $automate_Ncheck not in INIT.CHECK state" + end + break + end + + book getword automate $pageName check -var checkCommand + if (("$checkCommand" == "NULL") || ("$checkCommand" == "NONE")) + # if there is no check needed, we can immediate progress to the next stage (INIT.LAUNCH) + book setword automate $pageName pantaskState INIT.LAUNCH + if ($VERBOSE >= 2) + echo "$pageName is ready : INIT.LAUNCH" + end + break + end + + book setword automate $pageName pantaskState RUN.CHECK + + if ($VERBOSE >= 2) + echo "starting automate check for $pageName" + echo "command $checkCommand" + end + + options $pageName + command $checkCommand + end + + task.exit $EXIT_SUCCESS + local pageName Npage + + $pageName = $options:0 + + # convert 'stdout' to book format + # XXX to use other tests, we'll need to modify this + ipptool2book stdout tmpCheck + if ($VERBOSE > 2) + book listbook tmpCheck + end + + book getword automate $pageName ncheck -var Ncheck + if ("$Ncheck" == "NULL") + $Ncheck = 1 + end + book npages tmpCheck -var Npage + if ($Npage < $Ncheck) + if ($VERBOSE >= 2) + echo "$pageName not ready for LAUNCH" + end + book setword automate $pageName pantaskState INIT.CHECK + else + if ($VERBOSE >= 2) + echo "$pageName is ready for LAUNCH" + end + book setword automate $pageName pantaskState INIT.LAUNCH + + if ($Ncheck == 1) + # XXX this is a somewhat hackish way to carry information from the 'check' to the 'launch' + book getword tmpCheck page.000 det_id -var DET_ID + book getword tmpCheck page.000 iteration -var ITERATION + book setword automate $pageName det_id $DET_ID + book setword automate $pageName iteration $ITERATION + end + end + + # drop the detExt book after we've grabbed the state + book delete tmpCheck + end + + # all failures here (what state?) + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +$automate_Nlaunch = 0 +task automate.launch + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + task.exec + local Npage pageName + + # how many pages are waiting to be started? + book npages automate -var Npage -key pantaskState INIT.LAUNCH + if ($Npage == 0) + if ($VERBOSE >= 2) + echo "no entries in INIT.LAUNCH state" + end + break + end + + # cycle over the number of INIT.LAUNCH pages: no point to loop over the others + $automate_Nlaunch ++ + if ($automate_Nlaunch >= $Npage) set automate_Nlaunch = 0 + + # search the automate book for an entry which is unstarted (state INIT.LAUNCH) + book getpage automate $automate_Nlaunch -var pageName -key pantaskState INIT.LAUNCH + if ("$pageName" == "NULL") + if ($VERBOSE >= 2) + echo "entry $automate_Nlaunch not in INIT.LAUNCH state" + end + break + end + + book getword automate $pageName launch -var launchCommand + if (("$launchCommand" == "NULL") || ("$launchCommand" == "NONE")) + # if there is no launch needed, we can immediate progress to the next stage (INIT.LAUNCH) + book setword automate $pageName pantaskState INIT.LAUNCH + if ($VERBOSE >= 2) + echo "$pageName is ready : INIT.LAUNCH" + end + break + end + + # modify the launch command to replace certain elements from the page + book getword automate $pageName det_id -var DET_ID + if ("$DET_ID" != "NULL") + echo '$launchCommand' + strsub "$launchCommand" @det_id@ $DET_ID -var launchCommand + end + + book getword automate $pageName iteration -var ITERATION + if ("$ITERATION" != "NULL") + echo '$launchCommand' + strsub "$launchCommand" @iteration@ $ITERATION -var launchCommand + end + + echo '$launchCommand' + + book setword automate $pageName pantaskState RUN.LAUNCH + + if ($VERBOSE >= 2) + echo "starting automate launch for $pageName" + echo "command $launchCommand" + end + + options $pageName + command $launchCommand + end + + task.exit $EXIT_SUCCESS + local pageName Npage + + $pageName = $options:0 + + book setword automate $pageName pantaskState DONE.LAUNCH + end + + # all failures here (what state?) + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +$automate_Nregular = 0 +task automate.regular + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + task.exec + local Npage pageName + + # how many pages are waiting to be started? + book npages automate -var Npage -key pantaskState INIT.REGULAR + if ($Npage == 0) + if ($VERBOSE >= 2) + echo "no entries in INIT.REGULAR state" + end + break + end + + # cycle over the number of INIT.REGULAR pages: no point to loop over the others + $automate_Nregular ++ + if ($automate_Nregular >= $Npage) set automate_Nregular = 0 + + # search the automate book for an entry which is unstarted (state INIT.REGULAR) + book getpage automate $automate_Nregular -var pageName -key pantaskState INIT.REGULAR + if ("$pageName" == "NULL") + if ($VERBOSE >= 2) + echo "entry $automate_Nregular not in INIT.REGULAR state" + end + break + end + + book getword automate $pageName regular -var regularCommand + if (("$regularCommand" == "NULL") || ("$regularCommand" == "NONE")) + if ($VERBOSE >= 2) + echo "Warning: $pageName has no regular command" + end + break + end + + book setword automate $pageName pantaskState RUN.REGULAR + + if ($VERBOSE >= 2) + echo "starting automate regular for $pageName" + echo "command $regularCommand" + end + + options $pageName + command $regularCommand + end + + task.exit $EXIT_SUCCESS + local pageName Npage + + $pageName = $options:0 + + book setword automate $pageName pantaskState INIT.REGULAR + end + + # all failures here (what state?) + task.exit default + book setword automate $pageName pantaskState FAIL.REGULAR + showcommand failure + end + + # operation times out? + task.exit timeout + book setword automate $pageName pantaskState INIT.REGULAR + showcommand timeout + end +end + Index: /branches/eam_branches/eam_branch_20080918/ippTasks/calibration.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/calibration.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/calibration.pro (revision 22161) @@ -0,0 +1,173 @@ +## calibration.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the calibration stage +## these tasks use the book 'calBook' + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/calibration +exec mkdir -p $LOGSUBDIR + +book init calBook + +macro calibration.reset + book init calBook +end + +macro calibration.status + book listbook calBook +end + +macro calibration.on + task calibration.init + active true + end +end + +macro calibration.off + task calibration.init + active false + end +end + +# these variables will cycle through the known ippdb database names +$calInit_DB = 0 + +# we have three steps here: +# 1) get the list of dvo databases: caltool -dbs +# 2) define a new calibration run for each of the dvo dbs +# caltool + +# create new calibration entries for the currently known DVO databases +# run this multiple times once an hour - on pass for each db +task calibration.load + host local + + # check the list of available dvo dbs regularly + periods -poll 10 + periods -exec 900 + periods -timeout 60 + + # this is a strange construction; what is the purpose? + if ($DB:n == 0) + npending $DB:n + else + npending 1 + end + + # define the command (does not depend on previous queries) + if ($DB:n != 0) + command caltool -dbs -active true + else + # save the DB name for the exit tasks + # note that this DB name refers to the ippdb, not the dvodb + option $DB:$calInit_DB + command caltool -dbs -dbname $DB:$calInit_DB + $calInit_DB ++ + if ($calInit_DB >= $DB:n) set calInit_DB = 0 + end + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/calibration.log + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + # XXX have ippTools report the dbname? + ipptool2book stdout calBook -key dbname:cal_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook calBook + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup calBook + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# create new calibration entries for the currently known DVO databases +# run this multiple times once an hour - one pass for each db +# can we include information that the db has been updated without a recent cal analysis? +task calibration.resort + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 60 + # trange Hourly@00:00 Hourly@00:10 + # every hour on the hour + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/calibration.log + + task.exec + book npages calBook -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in calBook + # the sequencing in this task set is by the pantasksState (see notes.txt) + book getpage calBook 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # get the current ST: + $RAs = 15*($ST - 1) + $RAe = 15*($ST - 2) + $DECs = -90.0 + $DECe = +90.0 + + $REGION = "$RAs,$RAe:$DECs,$DECe" + + book setword calBook $pageName pantaskState RUN + book setword calBook $pageName region $REGION + + # XXX probably need to set the output / log based on WORKDIR... + book getword calBook $pageName cal_id -var ID + book getword calBook $pageName dvodb -var DVODB + book getword calBook $pageName dbname -var DBNAME + + # specify choice of remote host + # set a specific DVO host here + if ($PARALLEL) + host anyhost + else + host local + end + + # save the pageName for future reference below + options $pageName + + # XXX do we modify relphot to loop over the filters? + $run = calibrate_dvo.pl --cal_id $ID --dvodb $DVODB --region $REGION + add_standard_args run + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit calBook $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword calBook $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080918/ippTasks/camera.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/camera.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/camera.pro (revision 22161) @@ -0,0 +1,296 @@ +## camera.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the camera analysis stage +## these tasks use the book camPendingExp + +# test for required global variables +check.globals + +book init camPendingExp +book init camPendingCleanup + +macro camera.status + book listbook camPendingExp + book listbook camPendingCleanup +end + +macro camera.reset + book init camPendingExp + book init camPendingCleanup +end + +macro camera.on + task camera.exp.load + active true + end + task camera.exp.run + active true + end +end + +macro camera.off + task camera.exp.load + active false + end + task camera.exp.run + active false + end +end + +macro camera.cleanup.on + task camera.cleanup.load + active true + end + task camera.cleanup.run + active true + end +end + +macro camera.cleanup.off + task camera.cleanup.load + active false + end + task camera.cleanup.run + active false + end +end + +# this variable will cycle through the known database names +$camera_DB = 0 + +# select images ready for camera analysis +# new entries are added to camPendingExp +# skip already-present entries +task camera.exp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/camera.exp.log + + task.exec + $run = camtool -pendingexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$camera_DB + $run = $run -dbname $DB:$camera_DB + $camera_DB ++ + if ($camera_DB >= $DB:n) set camera_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout camPendingExp -key cam_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook camPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup camPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the cameraexposure script on pending images +task camera.exp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10 + + task.exec + book npages camPendingExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in camPendingExp (pantaskState == INIT) + book getpage camPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword camPendingExp $pageName pantaskState RUN + book getword camPendingExp $pageName camera -var CAMERA + book getword camPendingExp $pageName exp_tag -var EXP_TAG + book getword camPendingExp $pageName cam_id -var CAM_ID + book getword camPendingExp $pageName workdir -var WORKDIR_TEMPLATE + book getword camPendingExp $pageName dvodb -var DVODB + book getword camPendingExp $pageName dbname -var DBNAME + book getword camPendingExp $pageName reduction -var REDUCTION + book getword camPendingExp $pageName state -var RUN_STATE + + # specify choice of remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s/%s.cm.%s" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID + + stdout $LOGDIR/camera.exp.log + stderr $LOGDIR/camera.exp.log + + $run = camera_exp.pl --exp_tag $EXP_TAG --cam_id $CAM_ID --camera $CAMERA --outroot $outroot --redirect-output --run-state $RUN_STATE + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + if ("$DVODB" != "NULL") + $run = $run --dvodb $DVODB + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit camPendingExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword camPendingExp $options:0 pantaskState TIMEOUT + end +end + +# this variable will cycle through the known database names +$camera_cleanup_DB = 0 + +# select images ready for cam analysis +# new entries are added to camPendingImfile +# skip already-present entries +task camera.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/camera.cleanup.log + + task.exec + $run = camtool -pendingcleanuprun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$camera_cleanup_DB + $run = $run -dbname $DB:$camera_cleanup_DB + $camera_cleanup_DB ++ + if ($camera_cleanup_DB >= $DB:n) set camera_cleanup_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout camPendingCleanup -key cam_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook camPendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup camPendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task camera.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + book npages camPendingCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in camPendingCleanup (pantaskState == INIT) + book getpage camPendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword camPendingCleanup $pageName pantaskState RUN + book getword camPendingCleanup $pageName camera -var CAMERA + book getword camPendingCleanup $pageName state -var CLEANUP_MODE + book getword camPendingCleanup $pageName cam_id -var CAM_ID + book getword camPendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and cam (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/camera.cleanup.log + stderr $LOGDIR/camera.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage camera --stage_id $CAM_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit camPendingCleanup $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword camPendingCleanup $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080918/ippTasks/chip.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/chip.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/chip.pro (revision 22161) @@ -0,0 +1,298 @@ +## chip.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the phase 0 stage +## these tasks use the book chipPendingImfile + +## XXX the cleanup stage probably needs to mimic the chip processing stages, with an exposure and imfile layer +## can we define the cleanup stages in a separate file derived from this file? + +# test for required global variables +check.globals + +book init chipPendingImfile +book init chipPendingCleanup + +macro chip.status + book listbook chipPendingImfile + book listbook chipPendingCleanup +end + +macro chip.reset + book init chipPendingImfile + book init chipPendingCleanup +end + +macro chip.on + task chip.imfile.load + active true + end + task chip.imfile.run + active true + end +end + +macro chip.off + task chip.imfile.load + active false + end + task chip.imfile.run + active false + end +end + +macro chip.cleanup.on + task chip.cleanup.load + active true + end + task chip.cleanup.run + active true + end +end + +macro chip.cleanup.off + task chip.cleanup.load + active false + end + task chip.cleanup.run + active false + end +end + +# this variable will cycle through the known database names +$chip_DB = 0 + +# select images ready for chip analysis +# new entries are added to chipPendingImfile +# skip already-present entries +task chip.imfile.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/chip.imfile.log + + task.exec + $run = chiptool -pendingimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$chip_DB + $run = $run -dbname $DB:$chip_DB + $chip_DB ++ + if ($chip_DB >= $DB:n) set chip_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout chipPendingImfile -key chip_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook chipPendingImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup chipPendingImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the chip_imfile.pl script on pending images +task chip.imfile.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages chipPendingImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in chipPendingImfile (pantaskState == INIT) + book getpage chipPendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword chipPendingImfile $pageName pantaskState RUN + book getword chipPendingImfile $pageName camera -var CAMERA + book getword chipPendingImfile $pageName exp_id -var EXP_ID + book getword chipPendingImfile $pageName exp_tag -var EXP_TAG + book getword chipPendingImfile $pageName chip_id -var CHIP_ID + book getword chipPendingImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword chipPendingImfile $pageName class_id -var CLASS_ID + book getword chipPendingImfile $pageName uri -var URI + book getword chipPendingImfile $pageName dbname -var DBNAME + book getword chipPendingImfile $pageName reduction -var REDUCTION + book getword chipPendingImfile $pageName state -var RUN_STATE + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s/%s.ch.%s" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_ID + + stdout $LOGDIR/chip.imfile.log + stderr $LOGDIR/chip.imfile.log + + $run = chip_imfile.pl --threads @MAX_THREADS@ --exp_id $EXP_ID --chip_id $CHIP_ID --class_id $CLASS_ID --uri $URI --camera $CAMERA --run-state $RUN_STATE --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit chipPendingImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword chipPendingImfile $options:0 pantaskState TIMEOUT + end +end + +# this variable will cycle through the known database names +$chip_cleanup_DB = 0 + +# select images ready for chip analysis +# new entries are added to chipPendingImfile +# skip already-present entries +task chip.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/chip.cleanup.log + + task.exec + $run = chiptool -pendingcleanuprun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$chip_cleanup_DB + $run = $run -dbname $DB:$chip_cleanup_DB + $chip_cleanup_DB ++ + if ($chip_cleanup_DB >= $DB:n) set chip_cleanup_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout chipPendingCleanup -key chip_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook chipPendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup chipPendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task chip.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + book npages chipPendingCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in chipPendingCleanup (pantaskState == INIT) + book getpage chipPendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword chipPendingCleanup $pageName pantaskState RUN + book getword chipPendingCleanup $pageName camera -var CAMERA + book getword chipPendingCleanup $pageName state -var CLEANUP_MODE + book getword chipPendingCleanup $pageName chip_id -var CHIP_ID + book getword chipPendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/chip.cleanup.log + stderr $LOGDIR/chip.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage chip --stage_id $CHIP_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit chipPendingCleanup $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword chipPendingCleanup $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080918/ippTasks/chiphosts.manoa.config =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/chiphosts.manoa.config (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/chiphosts.manoa.config (revision 22161) @@ -0,0 +1,19 @@ +## basic chiphost table: use the defaults for everything + +chiphosts MULTI + +chiphosts METADATA + camera STR MEGACAM +END + +chiphosts METADATA + camera STR CFH12K +END + +chiphosts METADATA + camera STR GPC1 +END + +chiphosts METADATA + camera STR gpc1 +END Index: /branches/eam_branches/eam_branch_20080918/ippTasks/chiphosts.mhpcc.config =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/chiphosts.mhpcc.config (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/chiphosts.mhpcc.config (revision 22161) @@ -0,0 +1,172 @@ +chiphosts MULTI + +chiphosts METADATA + camera STR MEGACAM + ccd00 STR po00 + ccd01 STR po01 + ccd02 STR po02 + ccd03 STR po03 + ccd04 STR po04 + ccd05 STR po05 + ccd06 STR po06 + ccd07 STR po07 + ccd08 STR po08 + ccd09 STR po09 +END + +chiphosts METADATA + camera STR CFH12K + ccd00 STR po10 + ccd01 STR po11 + ccd02 STR po12 +END + +chiphosts METADATA + camera STR GPC1 + + XY01 STR ipp018 + XY02 STR ipp018 + XY03 STR ipp018 + XY04 STR ipp018 + XY05 STR ipp018 + XY06 STR ipp018 + + XY10 STR ipp006 + XY11 STR ipp006 + XY12 STR ipp006 + XY13 STR ipp006 + XY14 STR ipp006 + XY15 STR ipp006 + + XY16 STR ipp007 + XY17 STR ipp007 + XY20 STR ipp007 + XY21 STR ipp007 + XY22 STR ipp007 + XY23 STR ipp007 + + XY24 STR ipp008 + XY25 STR ipp008 + XY26 STR ipp008 + XY27 STR ipp008 + XY30 STR ipp008 + XY31 STR ipp008 + + XY32 STR ipp009 + XY33 STR ipp009 + XY34 STR ipp009 + XY35 STR ipp009 + XY36 STR ipp009 + XY37 STR ipp009 + + XY40 STR ipp010 + XY41 STR ipp010 + XY42 STR ipp010 + XY43 STR ipp010 + XY44 STR ipp010 + XY45 STR ipp010 + + XY46 STR ipp011 + XY47 STR ipp011 + XY50 STR ipp011 + XY51 STR ipp011 + XY52 STR ipp011 + XY53 STR ipp011 + + # This set should be going to ipp017 + XY54 STR ipp007 + XY55 S0R ipp009 + XY56 STR ipp010 + XY57 STR ipp011 + XY60 STR ipp020 + XY61 STR ipp021 + + XY62 STR ipp020 + XY63 STR ipp020 + XY64 STR ipp020 + XY65 STR ipp020 + XY66 STR ipp020 + XY67 STR ipp020 + + XY71 STR ipp021 + XY72 STR ipp021 + XY73 STR ipp021 + XY74 STR ipp021 + XY75 STR ipp021 + XY76 STR ipp021 +END + +chiphosts METADATA + camera STR gpc1 + + ota01 STR ipp018 + ota02 STR ipp018 + ota03 STR ipp018 + ota04 STR ipp018 + ota19 STR ipp018 + ota06 STR ipp018 + + ota10 STR ipp006 + ota11 STR ipp006 + ota12 STR ipp006 + ota13 STR ipp006 + ota14 STR ipp006 + ota15 STR ipp006 + + ota16 STR ipp007 + ota17 STR ipp007 + ota20 STR ipp007 + ota21 STR ipp007 + ota22 STR ipp007 + ota23 STR ipp007 + + ota24 STR ipp008 + ota25 STR ipp008 + ota26 STR ipp008 + ota27 STR ipp008 + ota30 STR ipp008 + ota31 STR ipp008 + + ota32 STR ipp009 + ota33 STR ipp009 + ota34 STR ipp009 + ota35 STR ipp009 + ota36 STR ipp009 + ota37 STR ipp009 + + ota40 STR ipp010 + ota41 STR ipp010 + ota42 STR ipp010 + ota43 STR ipp010 + ota44 STR ipp010 + ota45 STR ipp010 + + ota46 STR ipp011 + ota47 STR ipp011 + ota50 STR ipp011 + ota51 STR ipp011 + ota52 STR ipp011 + ota53 STR ipp011 + + # This set should be going to ipp017 + ota54 STR ipp007 + ota55 STR ipp009 + ota56 STR ipp010 + ota57 STR ipp011 + ota60 STR ipp020 + ota61 STR ipp021 + + ota62 STR ipp020 + ota63 STR ipp020 + ota64 STR ipp020 + ota65 STR ipp020 + ota66 STR ipp020 + ota67 STR ipp020 + + ota71 STR ipp021 + ota72 STR ipp021 + ota73 STR ipp021 + ota74 STR ipp021 + ota75 STR ipp021 + ota76 STR ipp021 +END Index: /branches/eam_branches/eam_branch_20080918/ippTasks/configure.ac =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/configure.ac (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/configure.ac (revision 22161) @@ -0,0 +1,14 @@ +AC_PREREQ(2.59) + +AC_INIT([ippTasks], [1.1.0], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([pantasks.pro]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_MAINTAINER_MODE + +AC_PROG_INSTALL + +AC_CONFIG_FILES([ + Makefile +]) +AC_OUTPUT Index: /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.correct.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.correct.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.correct.pro (revision 22161) @@ -0,0 +1,298 @@ +## detrend.correct.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend correction stage +## these tasks use the books detPendingCorrectedImfile and detPendingCorrectedExp + +# test for required global variables +check.globals + +book init detPendingCorrectedImfile +book init detPendingCorrectedExp + +macro detcorr.reset + book init detPendingCorrectedImfile + book init detPendingCorrectedExp +end + +macro detcorr.status + echo detPendingCorrectedImfile + book listbook detPendingCorrectedImfile + echo detPendingCorrectedExp + book listbook detPendingCorrectedExp +end + +macro detcorr.on + task detrend.correct.load + active true + end + task detrend.correct.run + active true + end + task detrend.correctexp.load + active true + end + task detrend.correctexp.run + active true + end +end + +macro detcorr.off + task detrend.correct.load + active false + end + task detrend.correct.run + active false + end + task detrend.correctexp.load + active false + end + task detrend.correctexp.run + active false + end +end + + +# these variables will cycle through the known database names +$detPendingCorrectedImfile_DB = 0 +$detPendingCorrectedExp_DB = 0 + +# select images ready for copy +# new entries are added to detPendingCorrectedImfile +# compare the new list with the ones already selected +task detrend.correct.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detcorr.load.log + + task.exec + $run = dettool -tocorrectedimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingCorrectedImfile_DB + $run = $run -dbname $DB:$detPendingCorrectedImfile_DB + $detPendingCorrectedImfile_DB ++ + if ($detPendingCorrectedImfile_DB >= $DB:n) set detPendingCorrectedImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingCorrectedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingCorrectedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingCorrectedImfile + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand + end +end + +# run detrend_correct_imfile.pl on pending images +task detrend.correct.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingCorrectedImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingCorrectedImfile + book getpage detPendingCorrectedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingCorrectedImfile $pageName pantaskState RUN + book getword detPendingCorrectedImfile $pageName det_id -var DET_ID + book getword detPendingCorrectedImfile $pageName exp_id -var EXP_ID + book getword detPendingCorrectedImfile $pageName class_id -var CLASS_ID + book getword detPendingCorrectedImfile $pageName exp_tag -var EXP_TAG XXX where does this come from + book getword detPendingCorrectedImfile $pageName det_type -var DET_TYPE XXX where does this come from + book getword detPendingCorrectedImfile $pageName camera -var CAMERA XXX where does this come from + book getword detPendingCorrectedImfile $pageName uri -var URI + book getword detPendingCorrectedImfile $pageName corr_id -var CORR_ID + book getword detPendingCorrectedImfile $pageName corr_type -var CORR_TYPE + book getword detPendingCorrectedImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingCorrectedImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detcorr.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID + + stdout $LOGDIR/detcorr.run.log + stderr $LOGDIR/detcorr.run.log + + $run = detrend_correct_imfile.pl --det_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --exp_tag $EXP_TAG --input_uri $URI --corr_id $CORR_ID --corr_type $CORR_TYPE --camera $CAMERA --outroot $outroot --redirect-output --verbose + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingCorrectedImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingCorrectedImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingCorrectedExp +# compare the new list with the ones already selected +task detrend.correctexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detcorr.exp.load.log + + task.exec + $run = dettool -tocorrectedexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingCorrectedExp_DB + $run = $run -dbname $DB:$detPendingCorrectedExp_DB + $detPendingCorrectedExp_DB ++ + if ($detPendingCorrectedExp_DB >= $DB:n) set detPendingCorrectedExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingCorrectedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingCorrectedExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingCorrectedExp + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.correctexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingCorrectedExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new exposures in detPendingCorrectedExp + book getpage detPendingCorrectedExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingCorrectedExp $pageName pantaskState RUN + book getword detPendingCorrectedExp $pageName det_id -var DET_ID + book getword detPendingCorrectedExp $pageName exp_id -var EXP_ID + book getword detPendingCorrectedExp $pageName det_type -var DET_TYPE + book getword detPendingCorrectedExp $pageName camera -var CAMERA + book getword detPendingCorrectedExp $pageName exp_tag -var EXP_TAG + book getword detPendingCorrectedExp $pageName corr_id -var CORR_ID + book getword detPendingCorrectedExp $pageName corr_type -var CORR_TYPE + book getword detPendingCorrectedExp $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingCorrectedExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $camera FPA + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detcorr.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID + + stdout $LOGDIR/detcorr.exp.run.log + stderr $LOGDIR/detcorr.exp.run.log + + $run = detrend_correct_exp.pl --det_id $DET_ID --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --corr_id $CORR_ID --corr_type $CORR_TYPE --camera $CAMERA --redirect-output --verbose + + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit detPendingCorrectedExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingCorrectedExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.mkruns.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.mkruns.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.mkruns.pro (revision 22161) @@ -0,0 +1,99 @@ +## detrend.mkruns.pro : globals and support macros : -*- sh -*- + +## XXX this script needs to be updated to use books, not queues +## XXX this script needs to be reconsidered... + +## This script defines tasks which regularly initiate the detrend +## runs. Each detrend type is given its own task, which are run once +## (or twice) per day to build the relevant detrend data + +## when one of these tasks is run, it creates a new detrend run ID in +## the database, and selects the exposures for that run on the basis +## of the specified filters + +# create a new detrend bias-creation run +task mkdetruns.megacam + # command dettool -select -time NOW -24h -type bias -camera megacam + command mkdetruns.megacam + host local + + periods -poll 1 + periods -exec 60 + periods -timeout 30 + trange 18:00 19:00 -nmax 1 + + # success: a new detrend run was created + task.exit 0 + echo "new MEGACAM detrend runs" + queueprint stdout + end + + # default exit status + task.exit default + echo "unable to define a new detrend run" + end + + # operation timed out? is the database down? + task.exit timeout + # send someone email? + echo "unable to define a new detrend run" + end +end + +# create a new detrend bias-creation run +task mkdetruns.isp + # command dettool -select -time NOW -24h -type bias -camera megacam + command mkdetruns.isp + host local + + periods -poll 1 + periods -exec 60 + periods -timeout 30 + trange 18:00 19:00 -nmax 1 + + # success: a new detrend run was created + task.exit 0 + echo "new ISP detrend runs" + queueprint stdout + end + + # default exit status + task.exit default + echo "unable to define a new detrend run" + end + + # operation timed out? is the database down? + task.exit timeout + # send someone email? + echo "unable to define a new detrend run" + end +end + +# create a new detrend bias-creation run +task mkdetruns.gpc + # command dettool -select -time NOW -24h -type bias -camera megacam + command mkdetruns.gpc + host local + + periods -poll 1 + periods -exec 60 + periods -timeout 30 + trange 18:00 19:00 -nmax 1 + + # success: a new detrend run was created + task.exit 0 + echo "new GPC detrend runs" + queueprint stdout + end + + # default exit status + task.exit default + echo "unable to define a new detrend run" + end + + # operation timed out? is the database down? + task.exit timeout + # send someone email? + echo "unable to define a new detrend run" + end +end Index: /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.norm.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.norm.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.norm.pro (revision 22161) @@ -0,0 +1,781 @@ +## detrend.norm.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend normalization stages +## these tasks use the books detPendingNormStatImfile detPendingNormImfile detPendingNormExp + +# test for required global variables +check.globals + +book init detPendingNormStatImfile +book init detPendingNormImfile +book init detPendingNormExp + +book init detCleanupNormStatImfile +book init detCleanupNormImfile +book init detCleanupNormExp + +macro detnorm.reset + book init detPendingNormStatImfile + book init detPendingNormImfile + book init detPendingNormExp + + book init detCleanupNormStatImfile + book init detCleanupNormImfile + book init detCleanupNormExp +end + +macro detnorm.status + book listbook detPendingNormStatImfile + book listbook detPendingNormImfile + book listbook detPendingNormExp + + book listbook detCleanupNormStatImfile + book listbook detCleanupNormImfile + book listbook detCleanupNormExp +end + +macro detnorm.on + task detrend.norm.load + active true + end + task detrend.norm.run + active true + end + task detrend.normexp.load + active true + end + task detrend.normexp.run + active true + end + task detrend.normstat.load + active true + end + task detrend.normstat.run + active true + end + + task detrend.cleanup.norm.load + active true + end + task detrend.cleanup.norm.run + active true + end + task detrend.cleanup.normexp.load + active true + end + task detrend.cleanup.normexp.run + active true + end + task detrend.cleanup.normstat.load + active true + end + task detrend.cleanup.normstat.run + active true + end +end + +macro detnorm.off + task detrend.norm.load + active false + end + task detrend.norm.run + active false + end + task detrend.normexp.load + active false + end + task detrend.normexp.run + active false + end + task detrend.normstat.load + active false + end + task detrend.normstat.run + active false + end + task detrend.cleanup.norm.load + active false + end + task detrend.cleanup.norm.run + active false + end + task detrend.cleanup.normexp.load + active false + end + task detrend.cleanup.normexp.run + active false + end + task detrend.cleanup.normstat.load + active false + end + task detrend.cleanup.normstat.run + active false + end +end + +# these variables will cycle through the known database names +$detPendingNormStatImfile_DB = 0 +$detPendingNormImfile_DB = 0 +$detPendingNormExp_DB = 0 + +$detCleanupNormStatImfile_DB = 0 +$detCleanupNormImfile_DB = 0 +$detCleanupNormExp_DB = 0 + +# select images ready for copy +# new entries are added to detPendingNormStatImfile +# compare the new list with the ones already selected +task detrend.normstat.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.normstat.log + + task.exec + $run = dettool -tonormalizedstat + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingNormStatImfile_DB + $run = $run -dbname $DB:$detPendingNormStatImfile_DB + $detPendingNormStatImfile_DB ++ + if ($detPendingNormStatImfile_DB >= $DB:n) set detPendingNormStatImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingNormStatImfile -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingNormStatImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingNormStatImfile + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.normstat.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingNormStatImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingNormStatImfile + book getpage detPendingNormStatImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingNormStatImfile $pageName pantaskState RUN + book getword detPendingNormStatImfile $pageName det_id -var DET_ID + book getword detPendingNormStatImfile $pageName det_type -var DET_TYPE + book getword detPendingNormStatImfile $pageName iteration -var ITERATION + book getword detPendingNormStatImfile $pageName camera -var CAMERA + book getword detPendingNormStatImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingNormStatImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s.%s.normstat.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION + + stdout $LOGDIR/detrend.normstat.log + stderr $LOGDIR/detrend.normstat.log + + $run = detrend_norm_calc.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE --outroot $outroot --redirect-output + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingNormStatImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingNormStatImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingNormImfile +# compare the new list with the ones already selected +task detrend.norm.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.norm.log + + task.exec + $run = dettool -tonormalize + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingNormImfile_DB + $run = $run -dbname $DB:$detPendingNormImfile_DB + $detPendingNormImfile_DB ++ + if ($detPendingNormImfile_DB >= $DB:n) set detPendingNormImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingNormImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingNormImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingNormImfile + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.norm.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingNormImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingNormImfile + book getpage detPendingNormImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingNormImfile $pageName pantaskState RUN + book getword detPendingNormImfile $pageName det_type -var DET_TYPE + book getword detPendingNormImfile $pageName camera -var CAMERA + book getword detPendingNormImfile $pageName uri -var URI + book getword detPendingNormImfile $pageName det_id -var DET_ID + book getword detPendingNormImfile $pageName iteration -var ITERATION + book getword detPendingNormImfile $pageName class_id -var CLASS_ID + book getword detPendingNormImfile $pageName norm -var NORM + book getword detPendingNormImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingNormImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this stack + sprintf outroot "%s/%s.%s.%s/%s.%s.norm.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION + + stdout $LOGDIR/detrend.norm.log + stderr $LOGDIR/detrend.norm.log + + $run = detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE --outroot $outroot --redirect-output --verbose + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingNormImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingNormImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingNormExp +# compare the new list with the ones already selected +task detrend.normexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.normexp.log + + task.exec + $run = dettool -tonormalizedexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingNormExp_DB + $run = $run -dbname $DB:$detPendingNormExp_DB + $detPendingNormExp_DB ++ + if ($detPendingNormExp_DB >= $DB:n) set detPendingNormExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingNormExp -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingNormExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingNormExp + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.normexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingNormExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingNormExp + book getpage detPendingNormExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingNormExp $pageName pantaskState RUN + book getword detPendingNormExp $pageName det_id -var DET_ID + book getword detPendingNormExp $pageName iteration -var ITERATION + book getword detPendingNormExp $pageName det_type -var DET_TYPE + book getword detPendingNormExp $pageName camera -var CAMERA + book getword detPendingNormExp $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingNormExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s.%s.normexp.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION + + stdout $LOGDIR/detrend.normexp.log + stderr $LOGDIR/detrend.normexp.log + + $run = detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE --outroot $outroot + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingNormExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingNormExp $options:0 pantaskState TIMEOUT + end +end + +########## cleanup normstat ########### +task detrend.cleanup.normstat.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.normstat.log + + task.exec + $run = dettool -pendingcleanup_normalizedstat + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupNormStatImfile_DB + $run = $run -dbname $DB:$detCleanupNormStatImfile_DB + $detCleanupNormStatImfile_DB ++ + if ($detCleanupNormStatImfile_DB >= $DB:n) set detCleanupNormStatImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupNormStatImfile -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupNormStatImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupNormStatImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.normstat.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupNormStatImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupNormStatImfile (pantaskState == INIT) + book getpage detCleanupNormStatImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupNormStatImfile $pageName pantaskState RUN + book getword detCleanupNormStatImfile $pageName det_id -var DET_ID + book getword detCleanupNormStatImfile $pageName iteration -var ITERATION + book getword detCleanupNormStatImfile $pageName camera -var CAMERA + book getword detCleanupNormStatImfile $pageName state -var CLEANUP_MODE + book getword detCleanupNormStatImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.normstat.log + stderr $LOGDIR/detrend.cleanup.normstat.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.normstat.imfile --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupNormStatImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupNormStatImfile $options:0 pantaskState TIMEOUT + end +end + +########## cleanup norm (normalized.imfile) ########### +task detrend.cleanup.norm.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.norm.log + + task.exec + $run = dettool -pendingcleanup_normalizedimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupNormImfile_DB + $run = $run -dbname $DB:$detCleanupNormImfile_DB + $detCleanupNormImfile_DB ++ + if ($detCleanupNormImfile_DB >= $DB:n) set detCleanupNormImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupNormImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupNormImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupNormImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.norm.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupNormImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupNormImfile (pantaskState == INIT) + book getpage detCleanupNormImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupNormImfile $pageName pantaskState RUN + book getword detCleanupNormImfile $pageName det_id -var DET_ID + book getword detCleanupNormImfile $pageName iteration -var ITERATION + book getword detCleanupNormImfile $pageName class_id -var CLASS_ID + book getword detCleanupNormImfile $pageName camera -var CAMERA + book getword detCleanupNormImfile $pageName state -var CLEANUP_MODE + book getword detCleanupNormImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.norm.log + stderr $LOGDIR/detrend.cleanup.norm.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.norm.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupNormImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupNormImfile $options:0 pantaskState TIMEOUT + end +end + +########## cleanup normexp ########### +task detrend.cleanup.normexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.normexp.cleanup.log + + task.exec + $run = dettool -pendingcleanup_normalizedexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupNormExp_DB + $run = $run -dbname $DB:$detCleanupNormExp_DB + $detCleanupNormExp_DB ++ + if ($detCleanupNormExp_DB >= $DB:n) set detCleanupNormExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupNormExp -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupNormExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupNormExp + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.normexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupNormExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupNormExp (pantaskState == INIT) + book getpage detCleanupNormExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupNormExp $pageName pantaskState RUN + book getword detCleanupNormExp $pageName det_id -var DET_ID + book getword detCleanupNormExp $pageName iteration -var ITERATION + book getword detCleanupNormExp $pageName camera -var CAMERA + book getword detCleanupNormExp $pageName state -var CLEANUP_MODE + book getword detCleanupNormExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.normexp.log + stderr $LOGDIR/detrend.cleanup.normexp.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.norm.exp --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupNormExp $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupNormExp $options:0 pantaskState TIMEOUT + end +end + Index: /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.process.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.process.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.process.pro (revision 22161) @@ -0,0 +1,540 @@ +## detrend.process.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend processing stage +## these tasks use the books detPendingProcessedImfile and detPendingProcessedExp + +# test for required global variables +check.globals + +book init detPendingProcessedImfile +book init detPendingProcessedExp +book init detCleanupProcessedImfile +book init detCleanupProcessedExp + +macro detproc.reset + book init detPendingProcessedImfile + book init detPendingProcessedExp + book init detCleanupProcessedImfile + book init detCleanupProcessedExp +end + +macro detproc.status + echo detPendingProcessedImfile + book listbook detPendingProcessedImfile + echo detPendingProcessedExp + book listbook detPendingProcessedExp + echo detCleanupProcessedImfile + book listbook detCleanupProcessedImfile + echo detCleanupProcessedExp + book listbook detCleanupProcessedExp +end + +macro detproc.on + task detrend.process.load + active true + end + task detrend.process.run + active true + end + task detrend.processexp.load + active true + end + task detrend.processexp.run + active true + end + task detrend.cleanup.process.load + active true + end + task detrend.cleanup.process.run + active true + end + task detrend.cleanup.processexp.load + active true + end + task detrend.cleanup.processexp.run + active true + end +end + +macro detproc.off + task detrend.process.load + active false + end + task detrend.process.run + active false + end + task detrend.processexp.load + active false + end + task detrend.processexp.run + active false + end + task detrend.cleanup.process.load + active false + end + task detrend.cleanup.process.run + active false + end + task detrend.cleanup.processexp.load + active false + end + task detrend.cleanup.processexp.run + active false + end +end + + +# these variables will cycle through the known database names +$detPendingProcessedImfile_DB = 0 +$detPendingProcessedExp_DB = 0 +$detCleanupProcessedImfile_DB = 0 +$detCleanupProcessedExp_DB = 0 + +# select images ready for copy +# new entries are added to detPendingProcessedImfile +# compare the new list with the ones already selected +task detrend.process.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.process.imfile.log + + task.exec + $run = dettool -toprocessedimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingProcessedImfile_DB + $run = $run -dbname $DB:$detPendingProcessedImfile_DB + $detPendingProcessedImfile_DB ++ + if ($detPendingProcessedImfile_DB >= $DB:n) set detPendingProcessedImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingProcessedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingProcessedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingProcessedImfile + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand + end +end + +# run detrend_process_imfile.ps on pending images +task detrend.process.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingProcessedImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingProcessedImfile + book getpage detPendingProcessedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingProcessedImfile $pageName pantaskState RUN + book getword detPendingProcessedImfile $pageName det_id -var DET_ID + book getword detPendingProcessedImfile $pageName det_type -var DET_TYPE + book getword detPendingProcessedImfile $pageName exp_id -var EXP_ID + book getword detPendingProcessedImfile $pageName class_id -var CLASS_ID + book getword detPendingProcessedImfile $pageName exp_tag -var EXP_TAG + book getword detPendingProcessedImfile $pageName uri -var URI + book getword detPendingProcessedImfile $pageName camera -var CAMERA + book getword detPendingProcessedImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingProcessedImfile $pageName dbname -var DBNAME + book getword detPendingProcessedImfile $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # see chip.pro for examples + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detproc.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID + + stdout $LOGDIR/detrend.process.imfile.log + stderr $LOGDIR/detrend.process.imfile.log + + $run = detrend_process_imfile.pl --threads @MAX_THREADS@ --det_id $DET_ID --exp_id $EXP_ID --det_type $DET_TYPE --class_id $CLASS_ID --exp_tag $EXP_TAG --input_uri $URI --camera $CAMERA --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingProcessedImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingProcessedImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingProcessedExp +# compare the new list with the ones already selected +task detrend.processexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.process.exp.log + + task.exec + $run = dettool -toprocessedexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingProcessedExp_DB + $run = $run -dbname $DB:$detPendingProcessedExp_DB + $detPendingProcessedExp_DB ++ + if ($detPendingProcessedExp_DB >= $DB:n) set detPendingProcessedExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingProcessedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingProcessedExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingProcessedExp + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.processexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingProcessedExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new exposures in detPendingProcessedExp + book getpage detPendingProcessedExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingProcessedExp $pageName pantaskState RUN + book getword detPendingProcessedExp $pageName det_id -var DET_ID + book getword detPendingProcessedExp $pageName exp_id -var EXP_ID + book getword detPendingProcessedExp $pageName det_type -var DET_TYPE + book getword detPendingProcessedExp $pageName camera -var CAMERA + book getword detPendingProcessedExp $pageName exp_tag -var EXP_TAG + book getword detPendingProcessedExp $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingProcessedExp $pageName dbname -var DBNAME + book getword detPendingProcessedExp $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detproc.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID + + stdout $LOGDIR/detrend.process.exp.log + stderr $LOGDIR/detrend.process.exp.log + + $run = detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output --verbose + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit detPendingProcessedExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingProcessedExp $options:0 pantaskState TIMEOUT + end +end + +######## cleanup process imfile ######## +task detrend.cleanup.process.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.process.imfile.log + + task.exec + $run = dettool -pendingcleanup_processedimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupProcessedImfile_DB + $run = $run -dbname $DB:$detCleanupProcessedImfile_DB + $detCleanupProcessedImfile_DB ++ + if ($detCleanupProcessedImfile_DB >= $DB:n) set detCleanupProcessedImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupProcessedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupProcessedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupProcessedImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.process.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupProcessedImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupProcessedImfile (pantaskState == INIT) + book getpage detCleanupProcessedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupProcessedImfile $pageName pantaskState RUN + book getword detCleanupProcessedImfile $pageName det_id -var DET_ID + book getword detCleanupProcessedImfile $pageName exp_id -var EXP_ID + book getword detCleanupProcessedImfile $pageName class_id -var CLASS_ID + book getword detCleanupProcessedImfile $pageName camera -var CAMERA + book getword detCleanupProcessedImfile $pageName state -var CLEANUP_MODE + book getword detCleanupProcessedImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.process.imfile.log + stderr $LOGDIR/detrend.cleanup.process.imfile.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.process.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupProcessedImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupProcessedImfile $options:0 pantaskState TIMEOUT + end +end + + +######## cleanup process exp ######## +task detrend.cleanup.processexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.process.exp.log + + task.exec + $run = dettool -pendingcleanup_processedexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupProcessedExp_DB + $run = $run -dbname $DB:$detCleanupProcessedExp_DB + $detCleanupProcessedExp_DB ++ + if ($detCleanupProcessedExp_DB >= $DB:n) set detCleanupProcessedExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupProcessedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupProcessedExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupProcessedExp + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.processexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupProcessedExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupProcessedExp (pantaskState == INIT) + book getpage detCleanupProcessedExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupProcessedExp $pageName pantaskState RUN + book getword detCleanupProcessedExp $pageName det_id -var DET_ID + book getword detCleanupProcessedExp $pageName exp_id -var EXP_ID + book getword detCleanupProcessedExp $pageName camera -var CAMERA + book getword detCleanupProcessedExp $pageName state -var CLEANUP_MODE + book getword detCleanupProcessedExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.process.exp.log + stderr $LOGDIR/detrend.cleanup.process.exp.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.process.exp --stage_id $DET_ID --exp_id $EXP_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupProcessedExp $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupProcessedExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.reject.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.reject.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.reject.pro (revision 22161) @@ -0,0 +1,150 @@ +## detrend.reject.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend processing stage +## these tasks use the book detRejectExp + +# test for required global variables +check.globals + +book init detRejectExp + +macro detreject.reset + book init detRejectExp +end + +macro detreject.status + book listbook detRejectExp +end + +macro detreject.on + task detrend.reject.load + active true + end + task detrend.reject.run + active true + end +end + +macro detreject.off + task detrend.reject.load + active false + end + task detrend.reject.run + active false + end +end + +# this variable will cycle through the known database names +$detRejectExp_DB = 0 + +# select images ready for copy +# new entries are added to detRejectExp +# compare the new list with the ones already selected +task detrend.reject.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.reject.log + + task.exec + $run = dettool -todetrunsummary + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detRejectExp_DB + $run = $run -dbname $DB:$detRejectExp_DB + $detRejectExp_DB ++ + if ($detRejectExp_DB >= $DB:n) set detRejectExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detRejectExp -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detRejectExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detRejectExp + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand failure + end +end + +# copy new images, sending job to desired host +task detrend.reject.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detRejectExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detRejectExp + book getpage detRejectExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detRejectExp $pageName pantaskState RUN + book getword detRejectExp $pageName det_id -var DET_ID + book getword detRejectExp $pageName iteration -var ITERATION + book getword detRejectExp $pageName det_type -var DET_TYPE + book getword detRejectExp $pageName mode -var MODE + book getword detRejectExp $pageName camera -var CAMERA + book getword detRejectExp $pageName workdir -var WORKDIR_TEMPLATE + book getword detRejectExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s.%s.%s.%s.detreject" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION + + stdout $LOGDIR/detrend.reject.log + stderr $LOGDIR/detrend.reject.log + + $run = detrend_reject_exp.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detRejectExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand failure + book setword detRejectExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.resid.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.resid.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.resid.pro (revision 22161) @@ -0,0 +1,543 @@ +## detrend.resid.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend processing stage +## these tasks use the books detPendingResidImfile and detPendingResidExp + +# test for required global variables +check.globals + +book init detPendingResidImfile +book init detPendingResidExp +book init detCleanupResidImfile +book init detCleanupResidExp + +macro detresid.reset + book init detPendingResidImfile + book init detPendingResidExp + book init detCleanupResidImfile + book init detCleanupResidExp +end + +macro detresid.status + book listbook detPendingResidImfile + book listbook detPendingResidExp + book listbook detCleanupResidImfile + book listbook detCleanupResidExp +end + +macro detresid.on + task detrend.resid.load + active true + end + task detrend.resid.run + active true + end + task detrend.residexp.load + active true + end + task detrend.residexp.run + active true + end + task detrend.cleanup.resid.load + active true + end + task detrend.cleanup.resid.run + active true + end + task detrend.cleanup.residexp.load + active true + end + task detrend.cleanup.residexp.run + active true + end +end + +macro detresid.off + task detrend.resid.load + active false + end + task detrend.resid.run + active false + end + task detrend.residexp.load + active false + end + task detrend.residexp.run + active false + end + task detrend.cleanup.resid.load + active false + end + task detrend.cleanup.resid.run + active false + end + task detrend.cleanup.residexp.load + active false + end + task detrend.cleanup.residexp.run + active false + end +end + +# these variables will cycle through the known database names +$detPendingResidImfile_DB = 0 +$detPendingResidExp_DB = 0 +$detCleanupResidImfile_DB = 0 +$detCleanupResidExp_DB = 0 + +# select images ready for copy +# new entries are added to detPendingResidImfile +# compare the new list with the ones already selected +task detrend.resid.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.resid.imfile.log + + task.exec + $run = dettool -toresidimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingResidImfile_DB + $run = $run -dbname $DB:$detPendingResidImfile_DB + $detPendingResidImfile_DB ++ + if ($detPendingResidImfile_DB >= $DB:n) set detPendingResidImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingResidImfile -key det_id:iteration:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingResidImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingResidImfile + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.resid.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingResidImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingResidImfile + book getpage detPendingResidImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingResidImfile $pageName pantaskState RUN + book getword detPendingResidImfile $pageName det_id -var DET_ID + book getword detPendingResidImfile $pageName exp_id -var EXP_ID + book getword detPendingResidImfile $pageName iteration -var ITERATION + book getword detPendingResidImfile $pageName det_type -var DET_TYPE + book getword detPendingResidImfile $pageName mode -var MODE + book getword detPendingResidImfile $pageName exp_tag -var EXP_TAG + book getword detPendingResidImfile $pageName class_id -var CLASS_ID + book getword detPendingResidImfile $pageName uri -var URI + book getword detPendingResidImfile $pageName det_uri -var DET_URI + book getword detPendingResidImfile $pageName ref_det_id -var REF_DET_ID + book getword detPendingResidImfile $pageName ref_iter -var REF_ITER + book getword detPendingResidImfile $pageName camera -var CAMERA + book getword detPendingResidImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingResidImfile $pageName dbname -var DBNAME + book getword detPendingResidImfile $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detresid.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID $ITERATION + + stdout $LOGDIR/detrend.resid.imfile.log + stderr $LOGDIR/detrend.resid.imfile.log + + $run = detrend_resid_imfile.pl --threads @MAX_THREADS@ --det_id $DET_ID --iteration $ITERATION --ref_det_id $REF_DET_ID --ref_iter $REF_ITER --exp_id $EXP_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE --outroot $outroot --redirect-output --verbose + + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create command + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingResidImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingResidImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingResidExp +# compare the new list with the ones already selected +task detrend.residexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.resid.exp.log + + task.exec + $run = dettool -toresidexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingResidExp_DB + $run = $run -dbname $DB:$detPendingResidExp_DB + $detPendingResidExp_DB ++ + if ($detPendingResidExp_DB >= $DB:n) set detPendingResidExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingResidExp -key det_id:iteration:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingResidExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingResidExp + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.residexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingResidExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingResidExp + book getpage detPendingResidExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingResidExp $pageName pantaskState RUN + book getword detPendingResidExp $pageName det_id -var DET_ID + book getword detPendingResidExp $pageName exp_id -var EXP_ID + book getword detPendingResidExp $pageName iteration -var ITERATION + book getword detPendingResidExp $pageName det_type -var DET_TYPE + book getword detPendingResidExp $pageName mode -var MODE + book getword detPendingResidExp $pageName exp_tag -var EXP_TAG + book getword detPendingResidExp $pageName include -var INCLUDE + book getword detPendingResidExp $pageName camera -var CAMERA + book getword detPendingResidExp $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingResidExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detresid.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID $ITERATION + + stdout $LOGDIR/detrend.resid.exp.log + stderr $LOGDIR/detrend.resid.exp.log + + $run = detrend_resid_exp.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output --verbose + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingResidExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingResidExp $options:0 pantaskState TIMEOUT + end +end + +######## cleanup resid imfile ######## +task detrend.cleanup.resid.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.resid.imfile.log + + task.exec + $run = dettool -pendingcleanup_residimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupResidImfile_DB + $run = $run -dbname $DB:$detCleanupResidImfile_DB + $detCleanupResidImfile_DB ++ + if ($detCleanupResidImfile_DB >= $DB:n) set detCleanupResidImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupResidImfile -key det_id:iteration:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupResidImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupResidImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.resid.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupResidImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupResidImfile (pantaskState == INIT) + book getpage detCleanupResidImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupResidImfile $pageName pantaskState RUN + book getword detCleanupResidImfile $pageName det_id -var DET_ID + book getword detCleanupResidImfile $pageName exp_id -var EXP_ID + book getword detCleanupResidImfile $pageName class_id -var CLASS_ID + book getword detCleanupResidImfile $pageName iteration -var ITERATION + book getword detCleanupResidImfile $pageName camera -var CAMERA + book getword detCleanupResidImfile $pageName state -var CLEANUP_MODE + book getword detCleanupResidImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.resid.imfile.log + stderr $LOGDIR/detrend.cleanup.resid.imfile.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.resid.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupResidImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupResidImfile $options:0 pantaskState TIMEOUT + end +end + + +######## cleanup resid exp ######## +task detrend.cleanup.residexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.resid.exp.log + + task.exec + $run = dettool -pendingcleanup_residexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupResidExp_DB + $run = $run -dbname $DB:$detCleanupResidExp_DB + $detCleanupResidExp_DB ++ + if ($detCleanupResidExp_DB >= $DB:n) set detCleanupResidExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupResidExp -key det_id:iteration:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupResidExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupResidExp + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.residexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupResidExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupResidExp (pantaskState == INIT) + book getpage detCleanupResidExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupResidExp $pageName pantaskState RUN + book getword detCleanupResidExp $pageName det_id -var DET_ID + book getword detCleanupResidExp $pageName exp_id -var EXP_ID + book getword detCleanupResidExp $pageName iteration -var ITERATION + book getword detCleanupResidExp $pageName camera -var CAMERA + book getword detCleanupResidExp $pageName state -var CLEANUP_MODE + book getword detCleanupResidExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.resid.exp.log + stderr $LOGDIR/detrend.cleanup.resid.exp.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.resid.exp --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupResidExp $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupResidExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.stack.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.stack.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/detrend.stack.pro (revision 22161) @@ -0,0 +1,280 @@ +## detrend.stack.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend stacking stage +## these tasks use the book detPendingStackedImfile + +# test for required global variables +check.globals + +book init detPendingStackedImfile +book init detCleanupStackedImfile + +macro detstack.reset + book init detPendingStackedImfile + book init detCleanupStackedImfile +end + +macro detstack.status + echo detPendingStackedImfile + book listbook detPendingStackedImfile + echo detCleanupStackedImfile + book listbook detCleanupStackedImfile +end + +macro detstack.on + task detrend.stack.load + active true + end + task detrend.stack.run + active true + end + task detrend.cleanup.stack.load + active true + end + task detrend.cleanup.stack.run + active true + end +end + +macro detstack.off + task detrend.stack.load + active false + end + task detrend.stack.run + active false + end + task detrend.cleanup.stack.load + active false + end + task detrend.cleanup.stack.run + active false + end +end + +# this variable will cycle through the known database names +$detPendingStackedImfile_DB = 0 +$detCleanupStackedImfile_DB = 0 + +# select images ready for detrend_stack.pl +# new entries are added to detPendingStackedImfile +# compare the new list with the ones already selected +task detrend.stack.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.stack.log + + task.exec + $run = dettool -tostacked + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingStackedImfile_DB + $run = $run -dbname $DB:$detPendingStackedImfile_DB + $detPendingStackedImfile_DB ++ + if ($detPendingStackedImfile_DB >= $DB:n) set detPendingStackedImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingStackedImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingStackedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingStackedImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.stack.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingStackedImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingStackedImfile + book getpage detPendingStackedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingStackedImfile $pageName pantaskState RUN + book getword detPendingStackedImfile $pageName det_id -var DET_ID + book getword detPendingStackedImfile $pageName iteration -var ITERATION + book getword detPendingStackedImfile $pageName det_type -var DET_TYPE + book getword detPendingStackedImfile $pageName class_id -var CLASS_ID + book getword detPendingStackedImfile $pageName camera -var CAMERA + book getword detPendingStackedImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingStackedImfile $pageName dbname -var DBNAME + book getword detPendingStackedImfile $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure + sprintf outroot "%s/%s.%s.%s/%s.%s.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION + + stdout $LOGDIR/detrend.stack.log + stderr $LOGDIR/detrend.stack.log + + $run = detrend_stack.pl --threads @MAX_THREADS@ --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingStackedImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingStackedImfile $options:0 pantaskState TIMEOUT + end +end + +########## cleanup stack ########### +task detrend.cleanup.stack.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.stack.log + + task.exec + $run = dettool -pendingcleanup_stacked + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupStackedImfile_DB + $run = $run -dbname $DB:$detCleanupStackedImfile_DB + $detCleanupStackedImfile_DB ++ + if ($detCleanupStackedImfile_DB >= $DB:n) set detCleanupStackedImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupStackedImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupStackedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupStackedImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.stack.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupStackedImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupStackedImfile (pantaskState == INIT) + book getpage detCleanupStackedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupStackedImfile $pageName pantaskState RUN + book getword detCleanupStackedImfile $pageName det_id -var DET_ID + book getword detCleanupStackedImfile $pageName iteration -var ITERATION + book getword detCleanupStackedImfile $pageName class_id -var CLASS_ID + book getword detCleanupStackedImfile $pageName camera -var CAMERA + book getword detCleanupStackedImfile $pageName state -var CLEANUP_MODE + book getword detCleanupStackedImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.stack.log + stderr $LOGDIR/detrend.cleanup.stack.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.stack.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupStackedImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupStackedImfile $options:0 pantaskState TIMEOUT + end +end + Index: /branches/eam_branches/eam_branch_20080918/ippTasks/diff.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/diff.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/diff.pro (revision 22161) @@ -0,0 +1,288 @@ +## diff.pro : image difference analysis : -*- sh -*- + +## This file contains panTasks definitions for performing the image differencing. +## After a difference (with associated diff_id) is defined, the difference is performed +## (tasks in diffSkyfile). + +# test for required global variables +check.globals + +### Initialise the books containing the tasks to do +book init diffSkyfile +book init diffCleanup + +### Database lists +$diffSkycell_DB = 0 +$diffCleanup_DB = 0 + +### Check status of diffing tasks +macro diff.status + book listbook diffSkyfile + book listbook diffCleanup +end + +### Reset diffing tasks +macro diff.reset + book init diffSkyfile + book init diffCleanup +end + +### Turn diffing tasks on +macro diff.on + task diff.skycell.load + active true + end + task diff.skycell.run + active true + end +end + +### Turn diffing tasks off +macro diff.off + task diff.skycell.load + active false + end + task diff.skycell.run + active false + end +end + +macro diff.cleanup.on + task diff.cleanup.load + active true + end + task diff.cleanup.run + active true + end +end + +macro diff.cleanup.off + task diff.cleanup.load + active false + end + task diff.cleanup.run + active false + end +end + + +### Load tasks for doing the differences +### Tasks are loaded into diffSkyfile. +task diff.skycell.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/diff.skycell.log + + task.exec + $run = difftool -todiffskyfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$diffSkycell_DB + $run = $run -dbname $DB:$diffSkycell_DB + $diffSkycell_DB ++ + if ($diffSkycell_DB >= $DB:n) set diffSkycell_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout diffSkyfile -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook diffSkyfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup diffSkyfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the diff overlaps +### Tasks are taken from diffSkyfile. +task diff.skycell.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages diffSkyfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in diffSkyfile (pantaskState == INIT) + book getpage diffSkyfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword diffSkyfile $pageName pantaskState RUN + book getword diffSkyfile $pageName diff_id -var DIFF_ID + book getword diffSkyfile $pageName tess_id -var TESS_DIR + book getword diffSkyfile $pageName skycell_id -var SKYCELL_ID + book getword diffSkyfile $pageName camera -var CAMERA + book getword diffSkyfile $pageName workdir -var WORKDIR_TEMPLATE + book getword diffSkyfile $pageName dbname -var DBNAME + + # set the host and workdir based on the skycell hash + set.host.for.skycell $SKYCELL_ID + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + # XXX old code: + # host anyhost + # $WORKDIR = $WORKDIR_TEMPLATE + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.dif.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $DIFF_ID + + stdout $LOGDIR/diff.skycell.log + stderr $LOGDIR/diff.skycell.log + + $run = diff_skycell.pl --threads @MAX_THREADS@ --diff_id $DIFF_ID --outroot $outroot --redirect-output + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit diffSkyfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword diffSkyfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for diff analysis +# new entries are added to diffPendingImfile +# skip already-present entries +task diff.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/diff.cleanup.log + + task.exec + $run = difftool -pendingcleanuprun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$diffCleanup_DB + $run = $run -dbname $DB:$diffCleanup_DB + $diffCleanup_DB ++ + if ($diffCleanup_DB >= $DB:n) set diffCleanup_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout diffCleanup -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook diffCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup diffCleanup + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task diff.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + book npages diffCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in diffCleanup (pantaskState == INIT) + book getpage diffCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword diffCleanup $pageName pantaskState RUN + book getword diffCleanup $pageName camera -var CAMERA + book getword diffCleanup $pageName state -var CLEANUP_MODE + book getword diffCleanup $pageName diff_id -var DIFF_ID + book getword diffCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and diff (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/diff.cleanup.log + stderr $LOGDIR/diff.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit diffCleanup $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword diffCleanup $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080918/ippTasks/fake.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/fake.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/fake.pro (revision 22161) @@ -0,0 +1,288 @@ +## fake.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the phase 0 stage +## these tasks use the book fakePendingImfile + +# test for required global variables +check.globals + +book init fakePendingImfile +book init fakePendingCleanup + +macro fake.status + book listbook fakePendingImfile + book listbook fakePendingCleanup +end + +macro fake.reset + book init fakePendingImfile + book init fakePendingCleanup +end + +macro fake.on + task fake.imfile.load + active true + end + task fake.imfile.run + active true + end +end + +macro fake.off + task fake.imfile.load + active false + end + task fake.imfile.run + active false + end +end + +macro fake.cleanup.on + task fake.cleanup.load + active true + end + task fake.cleanup.run + active true + end +end + +macro fake.cleanup.off + task fake.cleanup.load + active false + end + task fake.cleanup.run + active false + end +end + +# this variable will cycle through the known database names +$fakeImfile_DB = 0 + +# select images ready for fake analysis +# new entries are added to fakePendingImfile +# skip already-present entries +task fake.imfile.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/fake.imfile.log + + task.exec + $run = faketool -pendingimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$fakeImfile_DB + $run = $run -dbname $DB:$fakeImfile_DB + $fakeImfile_DB ++ + if ($fakeImfile_DB >= $DB:n) set fakeImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout fakePendingImfile -key fake_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook fakePendingImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup fakePendingImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the fake_imfile.pl script on pending images +task fake.imfile.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages fakePendingImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in fakePendingImfile (pantaskState == INIT) + book getpage fakePendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fakePendingImfile $pageName pantaskState RUN + book getword fakePendingImfile $pageName camera -var CAMERA + book getword fakePendingImfile $pageName exp_id -var EXP_ID + book getword fakePendingImfile $pageName exp_tag -var EXP_TAG + book getword fakePendingImfile $pageName fake_id -var FAKE_ID + book getword fakePendingImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword fakePendingImfile $pageName class_id -var CLASS_ID + book getword fakePendingImfile $pageName chip_path_base -var CHIPROOT + book getword fakePendingImfile $pageName cam_path_base -var CAMROOT + book getword fakePendingImfile $pageName dbname -var DBNAME + book getword fakePendingImfile $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and fake (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s/%s.fk.%s" $WORKDIR $EXP_TAG $EXP_TAG $FAKE_ID + + stderr $LOGDIR/fake.imfile.log + stderr $LOGDIR/fake.imfile.log + + $run = fake_imfile.pl --exp_id $EXP_ID --fake_id $FAKE_ID --class_id $CLASS_ID --chiproot=$CHIPROOT --camroot=$CAMROOT --camera $CAMERA --outroot $outroot + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit fakePendingImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword fakePendingImfile $options:0 pantaskState TIMEOUT + end +end + +# this variable will cycle through the known database names +$fake_cleanup_DB = 0 + +# select images ready for fake analysis +# new entries are added to fakePendingImfile +# skip already-present entries +task fake.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/fake.cleanup.log + + task.exec + $run = faketool -pendingcleanuprun + if ($DB:n == 0) + option DEFAULT + command + else + # save the DB name for the exit tasks + option $DB:$fake_cleanup_DB + $run = $run -dbname $DB:$fake_cleanup_DB + $fake_cleanup_DB ++ + if ($fake_cleanup_DB >= $DB:n) set fake_cleanup_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout fakePendingCleanup -key fake_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook fakePendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup fakePendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task fake.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + book npages fakePendingCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in fakePendingCleanup (pantaskState == INIT) + book getpage fakePendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fakePendingCleanup $pageName pantaskState RUN + book getword fakePendingCleanup $pageName camera -var CAMERA + book getword fakePendingCleanup $pageName state -var CLEANUP_MODE + book getword fakePendingCleanup $pageName fake_id -var FAKE_ID + book getword fakePendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and fake (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/fake.cleanup.log + stderr $LOGDIR/fake.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage fake --stage_id $FAKE_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit fakePendingCleanup $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword fakePendingCleanup $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080918/ippTasks/flatcorr.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/flatcorr.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/flatcorr.pro (revision 22161) @@ -0,0 +1,216 @@ +## flatcorr.pro : tasks to run flat-field correction : -*- sh -*- +## this file contains the tasks for running the flatcorr stage +## these tasks use the book 'flatcorrBook' + +# test for required global variables +check.globals + +book init flatcorrBook + +macro flatcorr.reset + book init flatcorrBook +end + +macro flatcorr.status + book listbook flatcorrBook +end + +macro flatcorr.on + task flatcorr.load + active true + end + task flatcorr.run + active true + end +end + +macro flatcorr.off + task flatcorr.load + active false + end + task flatcorr.run + active false + end +end + +# these variables will cycle through the known ippdb database names +$flatcorr_addcamera_DB = 0 +$flatcorr_pendingprocess_DB = 0 + +# a flat-field correction run is initiated (manually) with a command like: +# flatcorr -definebyquery -filter r ... +# the result is a flatcorrRun entry, a set of chipRun entries, a linking list in flatcorrChipList. +# the chipRun entries are all set to stop at the "chip" stage. + +# we also can define a run with: +# flatcorr -definerun +# flatcorr -addchip ... + +# as the chip analysis progresses, we need to occasionally migrate the +# completed chips to the camera stage by calling: +# flatcorr -addcamera + +# we wait for the completed chip and camera analysis and search for +# completed processing analysis with: +# flatcorr -pendingprocess + +# these define arguments to an analysis run. successful completion of +# the analysis is marked with: +# flatcorr -addprocess +# and failures with: +# flatcorr -addprocess -fault N + +# migrate complete flatcorr chips to the camera stage analysis +task flatcorr.addcamera + host local + + # check the list of pending flatcorr runs + periods -poll 10 + periods -exec 120 + periods -timeout 60 + npending 1 + + # define the command (does not depend on previous queries) + if ($DB:n == 0) + command flatcorr -addcamera + else + # save the DB name for the exit tasks + # note that this DB name refers to the ippdb, not the dvodb + option $DB:$flatcorr_addcamera_DB + command flatcorr -addcamera -dbname $DB:$flatcorr_addcamera_DB + $flatcorr_addcamera_DB ++ + if ($flatcorr_addcamera_DB >= $DB:n) set flatcorr_addcamera_DB = 0 + end + + # silently drop stdout + stdout NULL + stderr $LOGDIR/flatcorr.log + + # success (no action required) + task.exit $EXIT_SUCCESS + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# create new flatcorr entries for the currently known DVO databases +# run this multiple times once an hour - on pass for each db +task flatcorr.load + host local + + # check the list of pending flatcorr runs + periods -poll 10 + periods -exec 900 + periods -timeout 60 + npending 1 + + # define the command (does not depend on previous queries) + $run = flatcorr -pendingprocess + if ($DB:n != 0) + # save the DB name for the exit tasks + option $DB:$flatcorr_pendingprocess_DB + $run = $run -dbname $DB:$flatcorr_pendingprocess_DB + $flatcorr_pendingprocess_DB ++ + if ($flatcorr_pendingprocess_DB >= $DB:n) set flatcorr_pendingprocess_DB = 0 + end + add_poll_args run + command $run + + # silently drop stdout + stdout NULL + stderr $LOGDIR/flatcorr.log + + # success + task.exit $EXIT_SUCCESS + ipptool2book stdout flatcorrBook -key corr_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook flatcorrBook + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup flatcorrBook + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# check for flatcorr runs which are ready to go +task flatcorr.run + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 60 + + # silently drop stdout + stdout $LOGDIR/flatcorr.log + stderr $LOGDIR/flatcorr.log + + task.exec + book npages flatcorrBook -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new entries in flatcorrBook + book getpage flatcorrBook 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword flatcorrBook $pageName pantaskState RUN + + # XXX probably need to set the output / log based on WORKDIR... + book getword flatcorrBook $pageName corr_id -var ID + book getword flatcorrBook $pageName dvodb -var DVODB + book getword flatcorrBook $pageName region -var REGION + book getword flatcorrBook $pageName filter -var FILTER + book getword flatcorrBook $pageName dbname -var DBNAME + book getword flatcorrBook $pageName workdir -var WORKDIR + + # specify choice of remote host + # set a specific DVO host here + if ($PARALLEL) + host anyhost + else + host local + end + + # save the pageName for future reference below + options $pageName + + # XXX get these arguments right + $run = flatcorr_proc.pl --corr_id $ID --dvodb $DVODB --region $REGION --filter $FILTER + add_standard_args run + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit flatcorrBook $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword flatcorrBook $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080918/ippTasks/ipphosts.manoa.config =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/ipphosts.manoa.config (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/ipphosts.manoa.config (revision 22161) @@ -0,0 +1,19 @@ +## basic chiphost table: use the defaults for everything + +chiphosts MULTI + +chiphosts METADATA + camera STR MEGACAM +END + +chiphosts METADATA + camera STR CFH12K +END + +chiphosts METADATA + camera STR GPC1 +END + +chiphosts METADATA + camera STR gpc1 +END Index: /branches/eam_branches/eam_branch_20080918/ippTasks/ipphosts.mhpcc.config =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/ipphosts.mhpcc.config (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/ipphosts.mhpcc.config (revision 22161) @@ -0,0 +1,185 @@ +ipphosts MULTI + +ipphosts METADATA + camera STR skycell + count S32 8 + sky00 STR ipp006 + sky01 STR ipp007 + sky02 STR ipp008 + sky03 STR ipp009 + sky04 STR ipp010 + sky05 STR ipp011 + sky06 STR ipp020 + sky07 STR ipp021 +END + +ipphosts METADATA + camera STR MEGACAM + ccd00 STR po00 + ccd01 STR po01 + ccd02 STR po02 + ccd03 STR po03 + ccd04 STR po04 + ccd05 STR po05 + ccd06 STR po06 + ccd07 STR po07 + ccd08 STR po08 + ccd09 STR po09 +END + +ipphosts METADATA + camera STR CFH12K + ccd00 STR po10 + ccd01 STR po11 + ccd02 STR po12 +END + +ipphosts METADATA + camera STR GPC1 + + XY01 STR ipp018 + XY02 STR ipp018 + XY03 STR ipp018 + XY04 STR ipp018 + XY05 STR ipp018 + XY06 STR ipp018 + + XY10 STR ipp006 + XY11 STR ipp006 + XY12 STR ipp006 + XY13 STR ipp006 + XY14 STR ipp006 + XY15 STR ipp006 + + XY16 STR ipp007 + XY17 STR ipp007 + XY20 STR ipp007 + XY21 STR ipp007 + XY22 STR ipp007 + XY23 STR ipp007 + + XY24 STR ipp008 + XY25 STR ipp008 + XY26 STR ipp008 + XY27 STR ipp008 + XY30 STR ipp008 + XY31 STR ipp008 + + XY32 STR ipp009 + XY33 STR ipp009 + XY34 STR ipp009 + XY35 STR ipp009 + XY36 STR ipp009 + XY37 STR ipp009 + + XY40 STR ipp010 + XY41 STR ipp010 + XY42 STR ipp010 + XY43 STR ipp010 + XY44 STR ipp010 + XY45 STR ipp010 + + XY46 STR ipp011 + XY47 STR ipp011 + XY50 STR ipp011 + XY51 STR ipp011 + XY52 STR ipp011 + XY53 STR ipp011 + + # This set should be going to ipp017 + XY54 STR ipp007 + XY55 S0R ipp009 + XY56 STR ipp010 + XY57 STR ipp011 + XY60 STR ipp020 + XY61 STR ipp021 + + XY62 STR ipp020 + XY63 STR ipp020 + XY64 STR ipp020 + XY65 STR ipp020 + XY66 STR ipp020 + XY67 STR ipp020 + + XY71 STR ipp021 + XY72 STR ipp021 + XY73 STR ipp021 + XY74 STR ipp021 + XY75 STR ipp021 + XY76 STR ipp021 +END + +ipphosts METADATA + camera STR gpc1 + + ota01 STR ipp018 + ota02 STR ipp018 + ota03 STR ipp018 + ota04 STR ipp018 + ota19 STR ipp018 + ota06 STR ipp018 + + ota10 STR ipp006 + ota11 STR ipp006 + ota12 STR ipp006 + ota13 STR ipp006 + ota14 STR ipp006 + ota15 STR ipp006 + + ota16 STR ipp007 + ota17 STR ipp007 + ota20 STR ipp007 + ota21 STR ipp007 + ota22 STR ipp007 + ota23 STR ipp007 + + ota24 STR ipp008 + ota25 STR ipp008 + ota26 STR ipp008 + ota27 STR ipp008 + ota30 STR ipp008 + ota31 STR ipp008 + + ota32 STR ipp009 + ota33 STR ipp009 + ota34 STR ipp009 + ota35 STR ipp009 + ota36 STR ipp009 + ota37 STR ipp009 + + ota40 STR ipp010 + ota41 STR ipp010 + ota42 STR ipp010 + ota43 STR ipp010 + ota44 STR ipp010 + ota45 STR ipp010 + + ota46 STR ipp011 + ota47 STR ipp011 + ota50 STR ipp011 + ota51 STR ipp011 + ota52 STR ipp011 + ota53 STR ipp011 + + # This set should be going to ipp017 + ota54 STR ipp007 + ota55 STR ipp009 + ota56 STR ipp010 + ota57 STR ipp011 + ota60 STR ipp020 + ota61 STR ipp021 + + ota62 STR ipp020 + ota63 STR ipp020 + ota64 STR ipp020 + ota65 STR ipp020 + ota66 STR ipp020 + ota67 STR ipp020 + + ota71 STR ipp021 + ota72 STR ipp021 + ota73 STR ipp021 + ota74 STR ipp021 + ota75 STR ipp021 + ota76 STR ipp021 +END Index: /branches/eam_branches/eam_branch_20080918/ippTasks/isp.det =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/isp.det (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/isp.det (revision 22161) @@ -0,0 +1,91 @@ + +detRunDef MULTI + +detRunDef METADATA + detRunName STR BIAS + prereq STR NONE + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/bias -det_type bias -select_exp_type bias -select_dateobs_begin 2007-01-13T00:00:00" + dbname STR isp + pantaskState STR DONE + detRunID STR 1 +END + +detRunDef METADATA + detRunName STR DARK-warm + prereq STR BIAS + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/dark -det_type dark -select_exp_type dark -select_exp_time_min 0.5 -select_ccd_temp_min -27.5 -select_ccd_temp_max -24.0 -ccd_temp_min -27.5 -ccd_temp_max -24.0" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR DARK-cool + prereq STR BIAS + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/dark -det_type dark -select_exp_type dark -select_exp_time_min 0.5 -select_ccd_temp_max -27.5 -ccd_temp_max -27.5" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR SHUTTER + prereq STR "DARK-warm DARK-cool" + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/shutter -det_type shutter -select_exp_type skyflat -select_exp_time_min 0.05 -select_filter g -select_dateobs_begin 2007-01-14T04:00:00 -select_dateobs_end 2007-01-14T05:00:00 -select_sat_pixel_frac_max 0.1" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-g + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/g -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter g -filter g -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-r + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/r -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter r -filter r -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-i + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/i -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter i -filter i -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-z + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/z -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter z -filter z -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-y + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/y -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter y -filter y -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FRINGE-z + prereq STR FLAT-z + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/fringe/z -det_type fringe -select_exp_type object -select_exp_time_min 0.05 -select_filter z -filter z -select_dateobs_begin 2007-01-14T09:19:00 -select_dateobs_end 2007-01-18" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FRINGE-y + prereq STR FLAT-y + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/fringe/y -det_type fringe -select_exp_type object -select_exp_time_min 0.05 -select_filter y -filter y -select_dateobs_begin 2007-01-13 -select_dateobs_end 2007-01-18" + dbname STR isp + pantaskState STR NULL +END Index: /branches/eam_branches/eam_branch_20080918/ippTasks/magic.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/magic.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/magic.pro (revision 22161) @@ -0,0 +1,419 @@ +## magic.pro : support for the streak removal : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/magic +exec mkdir -p $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init magicToTree +book init magicToProcess +book init magicToMask + +### Database lists +$magicTree_DB = 0 +$magicProcess_DB = 0 +$magicMask_DB = 0 + +### Check status of tasks +macro magic.status + book listbook magicToTree + book listbook magicToProcess + book listbook magicToMask +end + +### Reset tasks +macro magic.reset + book init magicTree + book init magicProcess + book init magicMask +end + +### Turn tasks on +macro magic.on + task magic.tree.load + active true + end + task magic.tree.run + active true + end + task magic.process.load + active true + end + task magic.process.run + active true + end + task magic.mask.load + active true + end + task magic.mask.run + active true + end +end + +### Turn tasks off +macro magic.on + task magic.tree.load + active false + end + task magic.tree.run + active false + end + task magic.process.load + active false + end + task magic.process.run + active false + end + task magic.mask.load + active false + end + task magic.mask.run + active false + end +end + +task magic.tree.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/magic.tree.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command magictool -totree -limit 20 + else + # save the DB name for the exit tasks + option $DB:$magicTree_DB + command magictool -totree -limit 20 -dbname $DB:$magicTree_DB + $magicTree_DB ++ + if ($magicTree_DB >= $DB:n) set magicTree_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicTree -key magic_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicTree + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicTree + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task magic.tree.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages magicTree -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicTree 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicTree $pageName pantaskState RUN + book getword magicTree $pageName magic_id -var MAGIC_ID + book getword magicTree $pageName exp_id -var EXP_ID + book getword magicTree $pageName camera -var CAMERA + book getword magicTree $pageName workdir -var WORKDIR_TEMPLATE + book getword magicTree $pageName dbname -var DBNAME + book getword magicTree $pageName tess_id -var TESS_DIR + book getword magicTree $pageName ra -var RA + book getword magicTree $pageName decl -var DEC + +# set.host.for.camera $CAMERA $MAGIC_ID +# set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR + host anyhost + $WORKDIR = $WORKDIR_TEMPLATE + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s.mgc.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID + + ## generate output log based on filerule (convert the URI to a PATH) + $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $MAGIC_ID --basename $outroot` + if ("$logfile" == "") + echo "WARNING: logfile not defined in magic.tree.run" + break + end + + stdout $logfile + stderr $logfile + dirname $logfile -var outpath + mkdir $outpath + + $run = magic_tree.pl --magic_id $MAGIC_ID --camera $CAMERA --tess_id $TESS_DIR --ra $RA --dec $DEC --outroot $outroot --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit magicTree $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicTree $options:0 pantaskState TIMEOUT + end +end + +task magic.process.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/magic.process.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command magictool -toprocess -limit 20 + else + # save the DB name for the exit tasks + option $DB:$magicTree_DB + command magictool -toprocess -limit 20 -dbname $DB:$magicProcess_DB + $magicProcess_DB ++ + if ($magicProcess_DB >= $DB:n) set magicProcess_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicProcess -key magic_id:node -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicProcess + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicProcess + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task magic.process.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages magicProcess -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicProcess 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicTree $pageName pantaskState RUN + book getword magicTree $pageName magic_id -var MAGIC_ID + book getword magicTree $pageName exp_id -var EXP_ID + book getword magicTree $pageName node -var NODE + book getword magicTree $pageName camera -var CAMERA + book getword magicTree $pageName workdir -var WORKDIR_TEMPLATE + book getword magicTree $pageName dbname -var DBNAME + +# set.host.for.camera $CAMERA $MAGIC_ID +# set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR + host anyhost + $WORKDIR = $WORKDIR_TEMPLATE + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s.mgc.%s.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID $NODE + + ## generate output log based on filerule (convert the URI to a PATH) + $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $MAGIC_ID --basename $outroot` + if ("$logfile" == "") + echo "WARNING: logfile not defined in magic.process.run" + break + end + + stdout $logfile + stderr $logfile + dirname $logfile -var outpath + mkdir $outpath + + $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --outroot $outroot --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit magicProcess $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicProcess $options:0 pantaskState TIMEOUT + end +end + +task magic.mask.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/magic.mask.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command magictool -tomask -limit 20 + else + # save the DB name for the exit tasks + option $DB:$magicMask_DB + command magictool -tomask -limit 20 -dbname $DB:$magicMask_DB + $magicMask_DB ++ + if ($magicMask_DB >= $DB:n) set magicMask_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicMask -key magic_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicMask + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicMask + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task magic.mask.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages magicMask -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicMask 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicMask $pageName pantaskState RUN + book getword magicMask $pageName magic_id -var MAGIC_ID + book getword magicMask $pageName exp_id -var EXP_ID + book getword magicMask $pageName camera -var CAMERA + book getword magicMask $pageName workdir -var WORKDIR_TEMPLATE + book getword magicMask $pageName dbname -var DBNAME + +# set.host.for.camera $CAMERA $MAGIC_ID +# set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR + host anyhost + $WORKDIR = $WORKDIR_TEMPLATE + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s.mgc.%s.mask" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID + + ## generate output log based on filerule (convert the URI to a PATH) + $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $MAGIC_ID --basename $outroot` + if ("$logfile" == "") + echo "WARNING: logfile not defined in magic.tree.run" + break + end + + stdout $logfile + stderr $logfile + dirname $logfile -var outpath + mkdir $outpath + + $run = magic_mask.pl --magic_id $MAGIC_ID --camera $CAMERA --outroot $outroot --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit magicMask $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicMask $options:0 pantaskState TIMEOUT + end +end + Index: /branches/eam_branches/eam_branch_20080918/ippTasks/mkdetrend.config =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/mkdetrend.config (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/mkdetrend.config (revision 22161) @@ -0,0 +1,43 @@ + +detRunDef MULTI + +# add in the min number of input images and a time range +detRunDef METADATA + detRunName STR BIAS + prereq STR NONE + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type BIAS -select_exp_type BIAS" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR DARK + prereq STR BIAS + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type DARK -select_exp_type DARK" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR SHUTTER + prereq STR DARK + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-r + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-i + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter i -select_exp_type FLAT -select_filter i" + dbname STR eamtest + pantaskState STR NULL +END Index: /branches/eam_branches/eam_branch_20080918/ippTasks/mkdetrend.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/mkdetrend.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/mkdetrend.pro (revision 22161) @@ -0,0 +1,217 @@ +## mkdetrend.pro : autorun the detrend analysis : -*- sh -*- + +# detRun METADATA +# det_id STR 3 +# iteration S32 0 +# det_type STR SHUTTER +# mode STR master +# state STR run +# filelevel STR FPA +# END + +macro load.detRunDefs + if ($0 != 2) + echo "USAGE: load.detRunDefs (filename)" + break + end + queueload detruns -x "cat $MODULES:0/$1" + ipptool2book detruns detRunDef -key detRunName -uniq -setword pantaskState INIT + queuedelete detruns +end + +task detrun.launch + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + # each outstanding detrend run blocks the next one. only launch the next + # if the current one is done + task.exec + local i j N found missed detRunName prereq prereqList prereqState cmdOptions dbname + + # how many detRunDefs are waiting to be started? + book npages detRunDef -var N -key pantaskState INIT + # echo "detrun.launch N: $N" + if ($N == 0) break + + $found = 0 + + # search the detRunDef book for an entry which is unstarted (state NULL) + # and for which the dependencies are done + for i 0 $N + book getpage detRunDef $i -var detRunName -key pantaskState INIT + # echo "detrun.launch detRunName: $detRunName" + if ("$detRunName" == "NULL") break + + book getword detRunDef $detRunName prereq -var prereqList + if (("$prereqList" == "NULL") || ("$prereqList" == "NONE")) + $found = 1 + last + end + + list prereq -split $prereqList + $missed = 0 + for j 0 $prereq:n + # prereq must exist and be in state of DONE + book getword detRunDef $prereq:$j pantaskState -var prereqState + + # echo "detrun.launch prereqState: $prereqState" + # echo "detrun.launch detRunName: $detRunName" + if ("$prereqState" != "DONE") + $missed = 1 + last + end + end + if ($missed == 0) + $found = 1 + last + end + + if ($found == 0) + if ($VERBOSE >= 2) + echo "$detRunName not ready" + end + else + if ($VERBOSE >= 2) + echo "$detRunName ready" + end + end + end + + if ($found == 0) break + + # echo "detrun.launch detRunName: $detRunName" + + book getword detRunDef $detRunName options -var cmdOptions + book getword detRunDef $detRunName dbname -var dbname + book setword detRunDef $detRunName pantaskState INIT + + if ($VERBOSE >= 1) + echo "starting detrend $detRunName :" + echo "command dettool $cmdOptions -dbname $dbname" + end + + options $dbname $detRunName + command dettool $cmdOptions -dbname $dbname + end + + task.exit $EXIT_SUCCESS + local detRunID + + $dbname = $options:0 + $detRunName = $options:1 + + # convert 'stdout' to book format + ipptool2book stdout detRun -key det_id -uniq + if ($VERBOSE > 2) + book listbook detRun + end + + # we should have launched only a single detRun + book getpage detRun 0 -var detRunID + if ("$detRunID" == "NULL") + echo "ERROR:detRun not defined" + break + end + + # we need to track the detID generated + book setword detRunDef $detRunName detRunID $detRunID + book setword detRunDef $detRunName pantaskState RUN + + # drop the detRun book after we've grabbed the state + book delete detRun + end +end + +$detRunNcheck = 0 + +# loop over the outstanding detRuns and update state +task detrun.check + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + # each outstanding detrend run blocks the next one. only launch the next + # if the current one is done + task.exec + local i N found detRunName detRunID dbname + + book npages detRunDef -var N -key pantaskState RUN + if ($N == 0) break + if ($detRunNcheck >= $N) + $detRunNcheck = 0 + end + + # try the next detRun entry which is in state RUN + book getpage detRunDef $detRunNcheck -var detRunName -key pantaskState RUN + $detRunNcheck ++ + + if ("$detRunName" == "NULL") + if ($VERBOSE >= 3) + echo "no more active detruns (Ncheck = $detRunNcheck)" + end + break + end + + # info needed to check on current state + book getword detRunDef $detRunName detRunID -var detRunID + book getword detRunDef $detRunName dbname -var dbname + + if ($VERBOSE >= 2) + echo " checking on $detRunName" + echo "command dettool -runs -det_id $detRunID -dbname $dbname" + end + + # we need a dettool command which takes a specific det_id and returns + # the current state + options $dbname $detRunName + command dettool -runs -det_id $detRunID -dbname $dbname + end + + task.exit $EXIT_SUCCESS + local detRunID state + + $dbname = $options:0 + $detRunName = $options:1 + + # convert 'stdout' to book format + ipptool2book stdout detRun -key det_id -uniq + if ($VERBOSE > 2) + book listbook detRun + end + + # we should have launched only a single detRun + book getpage detRun 0 -var detRunID + if ("$detRunID" == "NULL") + echo "ERROR:detRun not defined" + break + end + + # what is the current state of the detRun? + book getword detRun $detRunID state -var state + + # use any other state? + if ("$state" == "stop") + # update the state for this detRunName, drop detRun page + book setword detRunDef $detRunName pantaskState DONE + if ($VERBOSE > 1) + echo "$detRunName is done" + end + else + if ($VERBOSE > 1) + echo "$detRunName is still running" + end + end + + # drop this book when completed + book delete detRun + end +end Index: /branches/eam_branches/eam_branch_20080918/ippTasks/notes.txt =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/notes.txt (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/notes.txt (revision 22161) @@ -0,0 +1,373 @@ + +2008.08.31 + + * flatcorr modes: + * definebyquery : select the matching rawExp entries and add them to chip table. this is functionally identical to chiptool -definebyquery, + except that it also adds entries in the flatcorr tables to track the processing + + + + +2008.08.22 + + issues related to creating and applying the flat-field correction in a simtest context + + * generating the fake data: + * ppSim must be called with a recipe in which the SCATTER.FRACTION is non-zero + * simtest thus needs to accept a ppsim_recipe and supply it to ppSimSequence + * ppSimSequence needs to accept the ppsim_recipe and supply it to the generated ppSim calls + + * first stage processing: + * we need to build flats which are not corrected (use the standard 'flat' type?) + * we need to apply flats which are not corrected + ** these can both be accomplished with the current DEFAULT reduction class + + * second stage processing: + * we need to apply flats which are corrected + ** define a reduction class of type FLATCORR which: + * modifies ppImage.config:DETREND.CONTRAINTS:FLAT:DETTYPE = FLAT_CORRECT + + ** adding a new detrend type: + + * add an entry to detrend_stack.pl FILERULES + * add an entry to detrend_stack.pl STATRECIPES + * add an entry to ppMerge/src/ppMergeArguments.c + * add an entry to ippconfig/recipes/ppMerge.config + + * add an entry to ippconfig/recipes/rejections.config + * add a section to ippconfig/recipes/reductionClasses.mdc + + * ippScripts/scripts/detrend_norm_apply.pl : entry in DETTYPE table + * ippScripts/scripts/detrend_norm_calc.pl : entry in NORMALIZE table + * ippScripts/scripts/detrend_resid_imfile.pl : entry in DETRENDS table + * ippScripts/scripts/detrend_stack.pl : entry in FILERULES and STATRECIPES tables + * ippScripts/scripts/detrend_resid_imfile.pl : entry in DETRENDS table + + * need a block in simtest.auto (or equiv) to check for when the chip processing / cam processing is complete + + * need to modify the detselect -inst arguments in simtest.auto (use @INST@ or equiv) + + * reductions currently being used: + ** CHIP -- chip-level analysis + ** JPEG.BIN1, JPEG.BIN2 -- camera jpeg terms + ** JPEG_BIN1_IMAGE_(dettype), JPEG_BIN2_IMAGE_(dettype) + ** JPEG_BIN1_RESID_(dettype), JPEG_BIN2_RESID_(dettype) + ** (dettype)_STACK, + ** (dettype)_PROCESS, + ** FAKEPHOT + +2008.01.22 + + proposal to clean scripts/tasks up somewhat wrt names: + + * scripts do NOT use workdir to change the workdir in the db + * thus, what is supplied is actually the outputRoot + * require the output root for the scripts + * have the calling tasks construct the outputRoot + ** replace -workdir with -outroot + + some filename rules : + + * the db entry WORKDIR is the most abstract version: + path://GPC-@HOST@/20080130 + file://data/@HOST@.0/gpc1/20080130 + neb:///@HOST@/gpc1/20080130 + neb://gpc1/20080130 + XXX have nebulous distinguish between neb:///host/path and + neb://path? + + * the db WORKDIR entry is carried from step to step unless + over-ridden with a -queue command + + * psLib-based binaries (ppImage, etc) may be passed URIs, but not + files with the @HOST@ abstraction + + * IPP perl scripts may be passed URIs, but not files with the @HOST@ + abstraction + + * nebulous : convert neb:///@HOST@/path/name + + *** examples: + + * this tells ppImage to use the ipp004 version for both input and output: + ppImage -file neb:///ipp004-v0/gpc1/20080130/o4414g0001o/o4414g0001o.XY01.fits neb:///ipp004-v0/gpc1/20080130/o4414g0001o/o4414g0001o.XY01 + + * we already know the hostname at this point; include it in the call + chip_imfile.pl --uri neb:///ipp004-v0/gpc1/20080130/o4414g0001o/o4414g0001o.XY01.fits --workdir neb:///ipp004-v0/gpc1/20080130/ + + + + + +2008.01.21 + + I'm working on the summit.copy.pro interaction with nebulous (and + those of the other ippTasks) and the assignment of jobs and data + locations to hosts on the basis of the chip/host relationship. I + have defined a table which relates the chip (class_id) to the + desired host name. There are a few places, and a few slightly + different ways, in which this gets used: + + * choice of host for command execution: + + every job which operates on a specific chip must be told the chip + host so it can run the job on that machine. the information is + supplied in pantasks to the job with the 'host' command. I've + made a pantasks macro 'set.host.by.camera' which takes the camera + and class_id, looks up the match in the chip/host table, and calls + the host command with the desired host. If there is no match + (camera not listed in chip/host table or chip not listed in + table), the host is set to 'anyhost', which randomly chooses one. + I have used this command for all tasks which generate a pcontrol + command. For the tasks which are performed at the camera level, + these are set to anyhost. For the tasks which operate on stacks, + we can use the same structure to associate the skycell with a + host. + + * specific output filenames + + In both ippTasks and ippScripts, there are references to files + which must correctly located on the host. + + * workdir choice of output directory. + + the + + * dsget output target: + + dsget is called by pantasks in the task 'dsget'. currently, dsget + accepts a --filename argument which specifies the output filename + to use. in the nebulous context, this is of the form + neb://top/next/stuff, without any host-specific information. To + target the file instance to a specific host, we pass dsget the + --volume (name) argument. The volumes refer to the specific + devices (potentially more than one per host), and the names can + then be chosen to include the host. + +2007.10.11 + + The calibration suite uses a database table of known/registered DVO + databases to be calibrated. The suite operates on a segment of the + sky in each run, nominally a section ~1h west of the current ST. + + caltool -pendingdbs returns a list of entries (mdc format), one for + each DVO database in the table. + + caltool -databases [-active] : returns the list of dvo databases + caltool -addcalrun : inserts the stats for a single calibration run + caltool -setstate (active/inactive) : modify the state of a dvo + database + caltool -adddatabase : insert a new database + + XXX a potential source of collisions. the books are marked by + unique keys in the db tables, but a single pantasks run may be + addressing multiple databases. IDs are not unique across + databases. we can solve this by including in the key the dbname + + the calibration suite runs the following tasks, in sequences on each + of the databases: + + * resort, relphot, uniphot, relastro.objects, relastro.images + + ** detrend correction: any detrend images which need to be corrected + get inserted in the detCorrectedExp table. + + dettool commands needed: + * add a detrend exposure (det_id, exp_id) to the detCorrectedExp table + * add a detrend imfile (det_id, exp_id, class_id) to the + detCorrectedImfile table + * select pending detCorrectedExp + * select pending detCorrectedImfiles + + +2007.01.08 + + I have updated pantasks to used named tags in named groups to make + life much easier when changing the schema of the db tables. The + function ipptool2book convernts the output from an ippTool query in + the form of a set of metadata structures (saved in stdout) into a + book format. + + dettool -tonormalizedstat DetrendNormStat + dettool -tonormalize DetrendNorm + dettool -tonormalizedexp DetrendNormExp + dettool -toprocessedimfile DetrendProcessImfiles + dettool -toprocessedexp DetrendProcessExposures + dettool -residdetrun DetrendRejectExp + dettool -toresidimfile DetrendResidImfiles + dettool -toresidexp DetrendResidExposures + dettool -tostacked DetrendStackClass + p0tool -pendingimfile Phase0Imfiles + p0tool -pendingexp Phase0Exposures + p2tool -pendingimfile Phase2Imfiles + p3tool -pendingexp Phase3Exposures + +2006.08.28 + +- ipptools need to use expID:camera:telescope as unique key (ie, not expID alone) +- p0tool -pendingimfile -simple needs to report the expKey (ie, not expID) +- + + +This directory contains the PanTasks scripts which define the steps +performed by the IPP. These scripts use the ippTools to decide what +images need to be processed, and to update the Metadata Database with +the current state of the analysis. The analysis is performed using +IPP programs which are distributed to the cluster computers as needed. + +1. The Detrend Creation Scripts + +The detrend creation process is among the more complex elements of the +IPP. There are N major steps which must be performed: + +1.1. Define New Detrend Runs + +The script 'detrend.mkruns.pro' defines tasks which initiate new +Detrend Runs. There is a separate task for each camera automatically +processed by the system (mkdetruns.CAMERA). These tasks are executed +on a regular, but infrequent basis. The tasks call external helper +programs which actually run the command dettools -define. The helper +programs are thus responsible for identifying all of the types of +detrend images to be run for a given camera, as well as the correct +set of filters to apply. These tasks are run locally and only affect +the database. PanTasks does not track the detrend runs which are +created. + +- we may need to define these within the PanTasks scripts +- how frequently are these tasks run (1/day?) +- if no raw images are selected by the query, no new detrun should be + created. +- we may need different schedules for the dark (bias/dark) and the + light (flat/fringe) data. And yet another schedule for sky data + (skyflat/skyfringe). + +1.2. Process new raw imfiles + +The script 'detrend.process.pro' defines tasks which processes the new +raw imfiles for any detrend run. This script defines two tasks, +'dettool.raw.load' and 'dettool.raw.process'. The first script +occasionally polls the Metadata Database to select raw imfiles which +have been identified as part of a detrend run. The identified raw +imfiles are added to an internal queue, DetrendImfilesToProcess. Any +raw imfile already in the list is ignored. The imfiles are added to +the queue with a state of NEW. The second task examines the queue for +imfiles with state NEW and defines a ppImage analysis job for that +imfile. The job is sent to pcontrol for processing on the distributed +cluster. At this point, the corresponding queue entry is set to the +state RUN. Upon success, the queue entry is set to DONE and the +database is notified of the successful operation. The DONE entries +are occasionally pruned from the queue; this must be done in the same +task as the initial query and the update of the list based on the +database results. + +- we need to define the columns in the output of dettool -raw +- we need a method to associate an imfile with a node in the cluster +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs +- we need a method to identify the correct recipe (function of type?) + +1.3. Stack new processed imfiles + +The script 'detrend.stack.pro' defines tasks which stack the newly +processed imfiles for any detrend run iteration. This script defines +two tasks, 'dettool.stack.load and 'dettool.stack.process. The first +script occasionally polls the Metadata Database to select class IDs +for detrend runs for which the input imfiles have all been processed. +The identified detRun/classID combination is added to an internal +queue, DetrendClassIDtoStack. The entries are added to the queue with +a state of NEW. The second task examines the queue for entries with a +state of NEW. It defines a ppMerge analysis job for that +detID/classID. The helper script selects the corresponding input +imfiles and passes them to ppMerge. The job is sent to pcontrol for +processing on the distributed cluster. At this point, the +corresponding queue entry is set to the state RUN. Upon success, the +queue entry is set to DONE and the database is notified of the +successful operation. The DONE entries are occasionally pruned from +the queue; this must be done in the same task as the initial query and +the update of the list based on the database results. + +- we need to define the columns in the output of dettool -chip -processed +- we need a method to associate a classID with a node in the cluster +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) +- we need a method to identify the correct recipe (function of type?) + + +1.4. Normalize stacked masters + +The script 'detrend.norm.pro' defines tasks which normalize the newly +processed stacks for any detrend run iteration. This script defines +two tasks, 'dettool.norm.load and 'dettool.norm.process. The first +script occasionally polls the Metadata Database to select detrun IDs +for which the classes have all been stacked. The identified +detRunID/iteration combination is added to an internal queue, +DetrendRunToNormalize. The entries are added to the queue with a +state of NEW. The second task examines the queue for entries with a +state of NEW. It defines a ppNorm analysis job for that +detID/iteration. The job is sent to pcontrol for processing on the +distributed cluster. At this point, the corresponding queue entry is +set to the state RUN. Upon success, the queue entry is set to DONE and +the database is notified of the successful operation. The DONE +entries are occasionally pruned from the queue; this must be done in +the same task as the initial query and the update of the list based on +the database results. + +- we need to define the columns in the output of dettool +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) +- we need a method to identify the correct recipe (function of type?) +- we need to define the ppNorm operation inputs and outputs better + + +1.5. Generate Resid Imfiles + +The script 'detrend.resid.pro' defines tasks which apply the newly +generated detrend image to the corresponding input images to generate +the residual images. This script defines two tasks, +'dettool.resid.load and 'dettool.resid.process. The first script +occasionally polls the Metadata Database to select imfiles for which +the master detrend images has been normalized. The identified imfiles +are added to an internal queue, DetrendResidImfiles. The entries are +added to the queue with a state of NEW. The second task examines the +queue for entries with a state of NEW. It defines a ppImage analysis +job for that imfile. The job is sent to pcontrol for processing on +the distributed cluster. At this point, the corresponding queue entry +is set to the state RUN. Upon success, the queue entry is set to DONE +and the database is notified of the successful operation. The DONE +entries are occasionally pruned from the queue; this must be done in +the same task as the initial query and the update of the list based on +the database results. + +- we need to define the columns in the output of dettool +- can we select the imfiles here without apriori knowledge that the + master has been normalized? +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) +- we need a method to identify the correct recipe (function of type?) +- we need a method to pass the matching detrend image to ppImage in + the correct context (ie, a bias needs to be supplied to the -bias element...) + +1.6. Assess Detrend Results + +The script 'detrend.assess.pro' defines tasks which assess the +residual statistics for completed detrend runs. This script defines +two tasks, 'dettool.assess.load and 'dettool.assess.process. The +first script occasionally polls the Metadata Database to select +detrunIDs/iterations for which the resid imfiles have all been +generated. The identified detruns are added to an internal queue, +DetrendRunToAssess. The entries are added to the queue with a state +of NEW. The second task examines the queue for entries with a state +of NEW. It defines an analysis job for that collection. The job is +sent to pcontrol for processing on the distributed cluster. At this +point, the corresponding queue entry is set to the state RUN. Upon +success, the queue entry is set to DONE and the database is notified +of the successful operation. The DONE entries are occasionally pruned +from the queue; this must be done in the same task as the initial +query and the update of the list based on the database results. + +- the assess tool needs to trigger a new run, if needed. +- what is the command to run on the resid statistics? +- we need to define the columns in the output of dettool +- can we select the detrend runs here? +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) Index: /branches/eam_branches/eam_branch_20080918/ippTasks/pantasks.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/pantasks.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/pantasks.pro (revision 22161) @@ -0,0 +1,494 @@ +## pantasks.pro : globals and support macros : -*- sh -*- + +# globals used to control system behavior : these should be in uppercase +$NEBULOUS = 0 +$NETWORK = 1 +$PARALLEL = 1 +$VERBOSE = 1 +$LABEL = "NONE" +$POLLLIMIT = 32 +$LOGDIR = `pwd` +$LOGDIR = $LOGDIR/pantasks_logs +mkdir $LOGDIR +$KEEP_FAILURES = 0 + +$LOADPOLL = 0.25 +$LOADEXEC = 5 +$RUNPOLL = 0.5 +$RUNEXEC = 1 + +$EXIT_SUCCESS = 0 +$EXIT_UNKNOWN_ERR = 1 +$EXIT_SYS_ERR = 2 +$EXIT_CONFIG_ERR = 3 +$EXIT_PROG_ERR = 4 +$EXIT_DATA_ERR = 5 + +# DB lists the database names in use; the default ipprc database +# is only used if DB:n is 0 +$DB:n = 0 + +# very basic values: set these with init.copy.mhpcc +$default_host = anyhost +$workdir_template = `pwd` + +# user function to add databases by name (avoids duplicates) +macro add.database + if ($0 != 2) + echo "USAGE: add.database (db)" + break + end + if ($?DB:n == 0) + list DB -add $1 + return + end + + $found = 0 + for i 0 $DB:n + if ($DB:$i == $1) + $found = 1 + echo "$DB:$i set" + last + end + end + + if ($found == 0) + list DB -add $1 + end +end + +macro del.database + if ($0 != 2) + echo "USAGE: del.database (db)" + break + end + if ($?DB:n == 0) + return + end + + list DB -del $1 +end + +macro init.isp + list DB -add isp +end + +macro init.essence + list DB -add essence_v2 +end + +macro init.simtest.local + list DB -add simtest + $PARALLEL = 0 + controller exit true +end + +macro init.simtest.default.db + $host = `hostname` + if ($PARALLEL) + controller exit true + controller host add $host + end +end + +macro init.simtest + list DB -add simtest + + $host = `hostname` + if ($PARALLEL) + controller exit true + controller host add $host + end +end + +# need the ability to activate or deactivate specific tasks... + +macro detrend.on + detproc.on + detstack.on + detnorm.on + detresid.on + detreject.on +end + +macro detrend.off + detproc.off + detstack.off + detnorm.off + detresid.off + detreject.off +end + +macro detrend.reset + detproc.reset + detstack.reset + detnorm.reset + detresid.reset + detreject.reset +end + +macro all.on + register.on + detrend.on + chip.on + camera.on + fake.on + warp.on + diff.on + stack.on +end + +macro all.off + register.off + detrend.off + chip.off + camera.off + fake.off + warp.off + diff.off + stack.off +end + +macro module.tasks + module register.pro + module detrend.process.pro + module detrend.stack.pro + module detrend.norm.pro + module detrend.resid.pro + module detrend.reject.pro + module chip.pro + module camera.pro + module fake.pro + module warp.pro + module diff.pro + module stack.pro +end + +macro showcommand + local n + + if ($VERBOSE) + $command = $taskarg:0 + for n 1 $taskarg:n + $command = $command $taskarg:$n + end + if ($0 == 2) + echo "" + echo "$1 for: $command" + echo "job exit status: $JOB_STATUS" + echo "job host: $JOB_HOSTNAME" + echo "job dtime: $JOB_DTIME" + else + echo "command: $command" + end + end +end + +# Add standard arguments to the poll functions +macro add_poll_args + if ($0 != 2) + echo "Must pass in the command of interest, and this function will supplement" + stop + end + + local command + $command = $$1 -limit $POLLLIMIT + + # Only process the data with the specified label. + if ($?LABEL && $LABEL != "NONE") + $command = $command -label $LABEL + end + + $$1 = $command +end + +macro add_standard_args + if ($0 != 2) + echo "Must pass in the command of interest, and this function will supplement" + stop + end + + local command + $command = $$1 + + if ($?DBNAME && "$DBNAME" != "DEFAULT") + $command = $command --dbname $DBNAME + end + if ($?NOOP != 0) + if ($NOOP != 0) + $command = $command --no-op + end + end + if ($?NOUPDATE != 0) + if ($NOUPDATE != 0) + $command = $command --no-update + end + end + $$1 = $command --verbose +end + + +macro process_exit + if ($0 != 4) + echo "USAGE: process_exit (bookName) (pageName) (exitCode)" + break + end + + $bookName = $1 + $pageName = $2 + $exitCode = $3 + + if ($VERBOSE > 4) + echo "*** exit status ***" + echo "JOB_STATUS: $JOB_STATUS" + echo "JOB_HOSTNAME: $JOB_HOSTNAME" + echo "JOB_DTIME: $JOB_DTIME" + echo "*** stdout ***" + queueprint stdout + echo "*** stderr ***" + queueprint stderr + end + + # success + if ($exitCode == $EXIT_SUCCESS) + # the handler scripts update DB the tables; here we just update the page + book setword $bookName $pageName pantaskState DONE + return + end + + # failure related to the data files + # jobs which result in DATAERR must have db pantaskState updated + if ($exitCode == $EXIT_DATA_ERR) + showcommand failure + book setword $bookName $pageName pantaskState DATA_ERR + return + end + + if ($VERBOSE) + echo "*** stdout ***" + queueprint stdout + echo "*** stderr ***" + queueprint stderr + end + + # failure related to the data files + if ($exitCode == $EXIT_SYS_ERR) + # stop + showcommand "system failure" + book setword $bookName $pageName pantaskState SYS_ERR + return + end + + # failure related to the data files + if ($exitCode == $EXIT_CONFIG_ERR) + # stop + showcommand "config error" + book setword $bookName $pageName pantaskState CONFIG_ERR + return + end + + # failure related to the data files + if ($exitCode == $EXIT_PROG_ERR) + # stop + showcommand "programming error" + book setword $bookName $pageName pantaskState PROG_ERR + return + end + + # any other exit status + showcommand failure + book setword $bookName $pageName pantaskState UNKNOWN_ERR +end + +## XXX for the moment, remove all errors +macro process_cleanup + if ($0 != 2) + echo "USAGE: process_cleanup (bookname)" + break + end + + book delpage $1 -key pantaskState DONE + + if (not($KEEP_FAILURES)) + book delpage $1 -key pantaskState SYS_ERR + book delpage $1 -key pantaskState DATA_ERR + book delpage $1 -key pantaskState PROG_ERR + book delpage $1 -key pantaskState CONFIG_ERR + book delpage $1 -key pantaskState UNKNOWN_ERR + end +end + +macro set.verbosity + if ($0 != 2) + echo "USAGE: set.verbosity (level)" + break + end + $VERBOSE = $1 +end + +macro show.variables + ?? +end + +macro set.workdir.by.camera + if ($0 != 6) + echo "USAGE: set.workdir.by.camera (camera) (class_id) (template) (default) (varname)" + break + end + + local host default template camera classID varname + $camera = $1 + $classID = $2 + $template = $3 + $default = $4 + $varname = $5 + + if ("$template" == "NULL") + $varname = $workdir_template + echo "WARNING: WORKDIR template not set. Defaulting to $workdir_template" + break + end + + # missing camera and/or ipphosts table results in host = NULL + book getword ipphosts $camera $classID -var host + + if ("$host" == "NULL") + strsub $template @HOST@ $default -var $varname + else + strsub $template @HOST@ $host -var $varname + end +end + +macro set.workdir.by.skycell + if ($0 != 5) + echo "USAGE: set.workdir.by.skycell (skycellID) (template) (default) (varname)" + break + end + + local host default template skycellID varname count + $skycellID = $1 + $template = $2 + $default = $3 + $varname = $4 + + if ("$template" == "NULL") + $varname = $workdir_template + echo "WARNING: WORKDIR template not set. Defaulting to $workdir_template" + break + end + + # get the folding count for this camera + book getword ipphosts skycell count -var count + if ("$count" == "NULL") + strsub $template @HOST@ $default -var $varname + return + end + + strhash $skycellID $count -var skyhash + sprintf skyname "sky%02d" $skyhash + + # missing ipphosts table results in host = NULL + book getword ipphosts skycell $skyname -var host + + if ("$host" == "NULL") + strsub $template @HOST@ $default -var $varname + else + strsub $template @HOST@ $host -var $varname + end +end + +macro set.host.for.camera + if ($0 != 3) + echo "USAGE: set.host.for.camera (camera) (class_id)" + break + end + + local host + + if (not($PARALLEL)) + host local + return + end + + # missing camera and/or ipphosts table results in host = NULL + book getword ipphosts $1 $2 -var host + + if ("$host" == "NULL") + host anyhost + else + host $host + end +end + +macro set.host.for.skycell + if ($0 != 2) + echo "USAGE: set.host.for.skycell (skycellID)" + break + end + + local skycellID varname count host skyname skyhash + $skycellID = $1 + + # get the folding count for this camera + book getword ipphosts skycell count -var count + if ("$count" == "NULL") + host anyhost + return + end + + strhash $skycellID $count -var skyhash + sprintf skyname "sky%02d" $skyhash + + # missing ipphosts table results in host = NULL + book getword ipphosts skycell $skyname -var host + + if ("$host" == "NULL") + host anyhost + else + host $host + end +end + +macro get.host.for.camera + if ($0 != 3) + echo "USAGE: get.host.for.camera (camera) (class_id)" + break + end + + if (not($PARALLEL)) + echo local + return + end + + # missing camera and/or ipphosts table results in host = NULL + book getword ipphosts $1 $2 -var host + + if ("$host" == "NULL") + echo anyhost + else + echo $host + end +end + +macro check.globals + if ($?NETWORK == 0) + echo "NETWORK not defined: load pantasks.pro first" + break + end + if ($?PARALLEL == 0) + echo "PARALLEL not defined: load pantasks.pro first" + break + end + if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break + end + if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break + end + if ($?KEEP_FAILURES == 0) + echo "KEEP_FAILURES not defined: load pantasks.pro first" + break + end +end Index: /branches/eam_branches/eam_branch_20080918/ippTasks/pstamp.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/pstamp.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/pstamp.pro (revision 22161) @@ -0,0 +1,355 @@ + +#$VERBOSE = 3 + +# these variables wil cycle through the known database names + +$pstampDS_DB = 0 +$pstampReq_DB = 0 +$pstampJob_DB = 0 +$pstampFin_DB = 0 + +macro pstamp.reset + book init pstampRequest + book init pstampJob + book init pstampFinish +end + +pstamp.reset + + +macro pstamp.on + task pstamp.request.find + active true + end + task pstamp.request.load + active true + end + task pstamp.request.run + active true + end + + task request.finish.load + active true + end + task request.finish.run + active true + end + + task pstamp.job.load + active true + end + task pstamp.job.run + active true + end +end + +macro pstamp.off + task pstamp.request.find + active false + end + task pstamp.request.load + active false + end + task pstamp.request.run + active false + end + task request.finish.load + active false + end + task request.finish.run + active false + end + + task pstamp.job.load + active false + end + task pstamp.job.run + active false + end +end + +task pstamp.request.find + host local + + periods -poll $LOADPOLL + periods -exec 10 + periods -timeout 30 + npending 1 + + task.exec + if ($DB:n == 0) + option DEFAULT + command pstamp_queue_requests.pl --limit 5 + else + option $DB:$pstampDS_DB + command pstamp_queue_requests.pl --limit 5 --dbname $DB:$pstampDS_DB --verbose + $pstampDS_DB ++ + if ($pstampDS_DB >= $DB:n) set pstampDS_DB = 0 + end + end + + task.exit $EXIT_SUCCESS + # echo nothing to do + end + + task.exit default + showcommand failure + end + + task.exit timeout + showcommand timeout + end +end + +task pstamp.request.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 300 + npending 1 + + task.exec + if ($DB:n == 0) + option DEFAULT + command pstamptool -pendingreq + else + option $DB:$pstampReq_DB + command pstamptool -pendingreq -dbname $DB:$pstampReq_DB + $pstampReq_DB ++ + if ($pstampReq_DB >= $DB:n) set pstampReq_DB = 0 + end + end + + task.exit $EXIT_SUCCESS + ipptool2book stdout pstampRequest -key req_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + echo starting request + book listbook pstampRequest + end + + process_cleanup pstampRequest + end + + task.exit default + showcommand failure + end + + task.exit timeout + showcommand timeout + end +end + +task pstamp.request.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + npending 1 + + task.exec + book npages pstampRequest -var N + if ($N == 0) break + + book getpage pstampRequest 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pstampRequest $pageName pantaskState RUN + book getword pstampRequest $pageName req_id -var REQ_ID + book getword pstampRequest $pageName dbname -var DBNAME + + $run = pstamp_parser_run.pl --req_id $REQ_ID + + add_standard_args run + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + + task.exit default +# echo pstamp.request.run task.exit status: $JOB_STATUS + process_exit pstampRequest $options:0 $JOB_STATUS + end + + task.exit timeout + showcommand timeout + book setword pstampRequest $options:0 pantaskState TIMEOUT + end +end + +task request.finish.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 300 + npending 1 + + task.exec + if ($DB:n == 0) + option DEFAULT + command pstamptool -completedreq + else + option $DB:$pstampFin_DB + command pstamptool -completedreq -dbname $DB:$pstampFin_DB + $pstampFin_DB ++ + if ($pstampFin_DB >= $DB:n) set pstampFin_DB = 0 + end + end + + task.exit $EXIT_SUCCESS + ipptool2book stdout pstampFinish -key req_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook pstampFinish + end + + process_cleanup pstampFinish + end + + task.exit default + showcommand failure + end + + task.exit timeout + showcommand timeout + end +end + +task request.finish.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + npending 1 + + task.exec + book npages pstampFinish -var N + if ($N == 0) break + + book getpage pstampFinish 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pstampFinish $pageName pantaskState RUN + book getword pstampFinish $pageName req_id -var REQ_ID + book getword pstampFinish $pageName uri -var URI + book getword pstampFinish $pageName dbname -var DBNAME + book getword pstampFinish $pageName reqType -var REQ_TYPE + book getword pstampFinish $pageName name -var REQ_NAME + book getword pstampFinish $pageName outProduct -var PRODUCT + + $run = request_finish.pl --req_id $REQ_ID --req_type $REQ_TYPE --req_file $URI --req_name $REQ_NAME --product $PRODUCT + + add_standard_args run + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + task.exit default + # echo request.finish.run task.exit status: $JOB_STATUS + process_exit pstampFinish $options:0 $JOB_STATUS + end + + task.exit timeout + showcommand timeout + book setword pstampFinish $options:0 pantaskState TIMEOUT.run + end +end + +task pstamp.job.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + if ($DB:n == 0) + option DEFAULT + command pstamptool -pendingjob -limit 10 + else + option $DB:$pstampJob_DB + command pstamptool -pendingjob -limit 10 -dbname $DB:$pstampJob_DB + $pstampJob_DB ++ + if ($pstampJob_DB >= $DB:n) set pstampJob_DB = 0 + end + end + + task.exit $EXIT_SUCCESS + # echo ipptool2book stdout pstampJob -key job_id -setword dbname $options:0 -setword pantaskState INIT + ipptool2book stdout pstampJob -key job_id -uniq -setword dbname $options:0 -setword pantaskState INIT + + book npages pstampJob -var N + if ($VERBOSE > 2) + book listbook pstampJob + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup pstampJob + end + + task.exit default + showcommand failure + end + + task.exit timeout + showcommand timeout + end +end + +task pstamp.job.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 1200 + npending 1 + + task.exec + book npages pstampJob -var N + if ($N == 0) break + + book getpage pstampJob 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + #echo pageName: $pageName + + book setword pstampJob $pageName pantaskState RUN + book getword pstampJob $pageName job_id -var JOB_ID + book getword pstampJob $pageName dbname -var DBNAME + + if ($VERBOSE > 1) + book listpage pstampJob $pageName + end + + + $run = pstamp_job_run.pl --job_id $JOB_ID + add_standard_args run + + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + + task.exit default + if ($VERBOSE > 1) + echo pstamp.job.run task.exit $JOB_ID status: $JOB_STATUS + end + process_exit pstampJob $options:0 $JOB_STATUS + end + + task.exit timeout + echo pstamp.job.run task.timeout $JOB_ID status: $JOB_STATUS + process_exit pstampJob $options:0 $JOB_STATUS + showcommand timeout + book setword pstampJob $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080918/ippTasks/register.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/register.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/register.pro (revision 22161) @@ -0,0 +1,308 @@ +## register.pro : tasks for image registration (insert into database) : -*- sh -*- +## this file contains the tasks for running the registration stage +## these tasks use the books regPendingImfile and regPendingExp + +# test for required global variables +check.globals + +book init regPendingImfile +book init regPendingExp + +macro register.reset + book init regPendingImfile + book init regPendingExp +end + +macro register.status + book listbook regPendingImfile + book listbook regPendingExp +end + +macro register.on + task register.imfile.load + active true + end + task register.imfile.run + active true + end + task register.exp.load + active true + end + task register.exp.run + active true + end +end + +macro register.off + task register.imfile.load + active false + end + task register.imfile.run + active false + end + task register.exp.load + active false + end + task register.exp.run + active false + end +end + +# these variables will cycle through the known database names +$regPendingImfile_DB = 0 +$regPendingExp_DB = 0 + +# select images ready for register analysis +# new entries are added to regPendingImfile +# compare the new list with the ones already selected +task register.imfile.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGDIR/register.imfile.load.log + + # select entries from the current DB; cycle to the next DB, if it exists + # iff the DB list is not set, use the value defined in .ipprc + task.exec + $run = regtool -pendingimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$regPendingImfile_DB + $run = $run -dbname $DB:$regPendingImfile_DB + $regPendingImfile_DB ++ + if ($regPendingImfile_DB >= $DB:n) set regPendingImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout regPendingImfile -key exp_id:tmp_class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + book shuffle regPendingImfile + if ($VERBOSE > 2) + book listbook regPendingImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup regPendingImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the register_imfile.pl script on pending images +task register.imfile.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + if ($NETWORK == 0) break + book npages regPendingImfile -var N + if ($N == 0) + periods -exec $RUNEXEC + break + end + periods -exec 0.05 + + # look for new images in regPendingImfile + book getpage regPendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword regPendingImfile $pageName pantaskState RUN + + book getword regPendingImfile $pageName exp_id -var EXP_ID + book getword regPendingImfile $pageName tmp_exp_name -var TMP_EXP_NAME + book getword regPendingImfile $pageName tmp_camera -var TMP_CAMERA + book getword regPendingImfile $pageName tmp_class_id -var TMP_CLASS_ID + book getword regPendingImfile $pageName uri -var URI + book getword regPendingImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword regPendingImfile $pageName dbname -var DBNAME + + # EXP_TAG is used to generate the unique, but human-readable, filenames + sprintf EXP_TAG "%s.%s" $TMP_EXP_NAME $EXP_ID + + # specify choice of remote host + set.host.for.camera $TMP_CAMERA $TMP_CLASS_ID + + # set the WORKDIR variable + set.workdir.by.camera $TMP_CAMERA $TMP_CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + ## generate outroot specific to this exposure (& chip) + sprintf logfile "%s/%s/%s.reg.%s.log" $WORKDIR $EXP_TAG $EXP_TAG $TMP_CLASS_ID + + stdout $LOGDIR/register.imfile.run.log + stderr $LOGDIR/register.imfile.run.log + + # XXX register_imfile.pl differs from the standard script : it does not have an 'outroot' argument, and it does not take '--redirect' + $run = register_imfile.pl --exp_id $EXP_ID --tmp_class_id $TMP_CLASS_ID --tmp_exp_name $TMP_EXP_NAME --uri $URI --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit regPendingImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword regPendingImfile $options:0 pantaskState TIMEOUT + end +end + +# select exposures ready for register_exp.pl +task register.exp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/register.exp.load.log + + task.exec + $run = regtool -pendingexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$regPendingExp_DB + $run = $run -dbname $DB:$regPendingExp_DB + $regPendingExp_DB ++ + if ($regPendingExp_DB >= $DB:n) set regPendingExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout regPendingExp -key exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook regPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup regPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the register_exp.pl script on pending exposures +task register.exp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages regPendingExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in regPendingExp (pantaskState == INIT) + book getpage regPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword regPendingExp $pageName pantaskState RUN + book getword regPendingExp $pageName exp_id -var EXP_ID + book getword regPendingExp $pageName tmp_exp_name -var TMP_EXP_NAME + book getword regPendingExp $pageName tmp_camera -var TMP_CAMERA + book getword regPendingExp $pageName workdir -var WORKDIR_TEMPLATE + book getword regPendingExp $pageName dbname -var DBNAME + + # EXP_TAG is used to generate the unique, but human-readable, filenames + sprintf EXP_TAG "%s.%s" $TMP_EXP_NAME $EXP_ID + + # specify choice of remote host + set.host.for.camera $TMP_CAMERA $TMP_CLASS_ID + + # set the WORKDIR variable + set.workdir.by.camera $TMP_CAMERA $TMP_CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + ## generate output log based on filerule + sprintf logfile "%s/%s/%s.reg.log" $WORKDIR $EXP_TAG $EXP_TAG + + stdout $LOGDIR/register.exp.run.log + stderr $LOGDIR/register.exp.run.log + + $run = register_exp.pl --exp_id $EXP_ID --exp_tag $EXP_TAG --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit regPendingExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword regPendingExp $options:0 pantaskState TIMEOUT + end +end + +## XXX add a global path to output files Index: /branches/eam_branches/eam_branch_20080918/ippTasks/replicate.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/replicate.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/replicate.pro (revision 22161) @@ -0,0 +1,168 @@ +## replicate.pro : tasks for data replication : -*- sh -*- +## this file contains the tasks for maintaining duplicates as needed in Nebulous +## these tasks use the books replicatePending + +# test for required global variables +check.globals + +$NEB_DB = nebulous +$NEB_HOST = alala +$NEB_USER = jhipp +$NEB_PASS = jhipp + +$LOGSUBDIR = $LOGDIR/replicate +exec mkdir -p $LOGSUBDIR + +book init replicatePending + +macro replicate.reset + book init replicatePending +end + +macro replicate.status + book listbook replicatePending +end + +macro replicate.on + task replicate.load + active true + end + task replicate.run + active true + end +end + +macro replicate.off + task replicate.load + active false + end + task replicate.run + active false + end +end + +macro set.host.for.replicate + if ($0 != 2) + echo "USAGE: set.host.for.replicate (hostname)" + break + end + + if (not($PARALLEL)) + host local + return + end + +# parse volume name + + if ("$1" == "NULL") + host anyhost + else + host $1 + end +end + +# the replicate process interacts with only the single Nebulous server + +# select Nebulous objects which desire additional copies +task replicate.load + host local + + # modify these after the tasks are tested + periods -poll 10 + periods -exec 1 + periods -timeout 500 + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/replicate.log + + task.exec + # command does not need to be dynamic, but having it so allows us to adjust the periods + # so that we dont have to wait 10 minutes for things to start up + command neb-admin --host $NEB_HOST --db $NEB_DB --user $NEB_USER --pass $NEB_PASS --pendingreplicate --limit 2000 + periods -exec 600 + + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout replicatePending -key key -uniq -setword pantaskState INIT + + if ($VERBOSE > 2) + book listbook replicatePending + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup replicatePending + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# create the desired replicas +task replicate.run + periods -poll $RUNPOLL + periods -exec 30 + periods -timeout 30 + + task.exec + book npages replicatePending -var N + if ($NETWORK == 0) break + if ($N == 0) + periods -exec 30 + break + end + periods -exec 0.05 + + # look for new objects in replicatePending + book getpage replicatePending 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword replicatePending $pageName pantaskState RUN + + # XXX what values do I need to get back? + book getword replicatePending $pageName key -var KEY + book getword replicatePending $pageName need_copies -var NEED_COPIES + book getword replicatePending $pageName volume_name -var VOLUME_NAME + book getword replicatePending $pageName volume_host -var VOLUME_HOST + + set.host.for.replicate $VOLUME_HOST + + stdout NULL + stderr $LOGSUBDIR/replicate.log + + # these operations do not require a database to be specified + $run = neb-replicate --copies $NEED_COPIES $KEY + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit replicatePending $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword replicatePending $options:0 pantaskState TIMEOUT + end +end + Index: /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.auto =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.auto (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.auto (revision 22161) @@ -0,0 +1,119 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis + +automate METADATA + name STR BIAS + check STR "regtool -processedexp -exp_type BIAS -inst SIMTEST -dbname @DBNAME@" + ncheck S32 20 + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst SIMTEST -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@" +END + +automate METADATA + name STR DARK + check STR "detselect -search -inst SIMTEST -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst SIMTEST -det_type DARK -select_exp_type DARK -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -dbname @DBNAME@" +END + +automate METADATA + name STR SHUTTER + check STR "detselect -search -inst SIMTEST -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst SIMTEST -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-r + check STR "detselect -search -inst SIMTEST -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst SIMTEST -det_type FLAT -filter r -select_exp_type FLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-i + check STR "detselect -search -inst SIMTEST -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst SIMTEST -det_type FLAT -filter i -select_exp_type FLAT -select_filter i -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter i -dbname @DBNAME@" +END + +automate METADATA + name STR OBJECT-i + check STR "detselect -dbname @DBNAME@ -search -inst SIMTEST -filter i -det_type FLAT" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst SIMTEST -filter i -label wait -set_label proc" + block STR NONE +END + +automate METADATA + name STR OBJECT-r + check STR "detselect -dbname @DBNAME@ -search -inst SIMTEST -filter r -det_type FLAT" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst SIMTEST -filter r -label wait -set_label proc" + block STR NONE +END + +### Stack automation??? +automate METADATA + name STR STACK + regular STR "stacktool -definebyquery -all -workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@" +END + +### Diff automation??? +automate METADATA + name STR DIFF + regular STR "difftool -definebyquery -workdir file://@CWD@/diff -good_frac 0.2 -dbname @DBNAME@" +END + +### Magic automation??? +automate METADATA + name STR MAGIC + regular STR "magictool -definebyquery -workdir file://@CWD@/magic -good_frac 0.2 -dbname @DBNAME@" +END + + + + +### automate METADATA +### name STR OBJECT-r +### check STR "detselect -search -inst SIMTEST -det_type FLAT -dbname @DBNAME@ -filter r" +### launch STR "chiptool -unblock -label simtest-r -dbname @DBNAME@" +### block STR "chiptool -unmasked -label simtest-r -dbname @DBNAME@" +### END + +### automate METADATA +### name STR OBJECT-i +### check STR "detselect -search -inst SIMTEST -det_type FLAT -dbname @DBNAME@ -filter i" +### launch STR "chiptool -unblock -label simtest-i -dbname @DBNAME@" +### block STR "chiptool -unmasked -label simtest-i -dbname @DBNAME@" +### END + +### there is a weakness in the label / block business: the labels are +### generic, and the blocks are against those fairly generic words. +### that makes it difficult to block and unblock science exposures +### based on different detrend types that are available. For example, +### to block the r-band against the absence of the desired r-band +### flat, we would need to specify a label specific to the r-band +### images and remove that block when ready. + +### a better approach might be to modify the labels rather than the +### blocks. we can define, eg, chiptool -set_label -definebyquery to +### turn assign the label names based on various properties of the +### interesting images. + +### here is what the automate element might look like for such a circumstance: + +### automate METADATA +### name STR OBJECT-i +### block STR "chiptool -dbname @DBNAME@ -unmasked -label wait -filter i -time_begin 2008/1/1 -time_end 2008/1/2" +### check STR "detselect -dbname @DBNAME@ -search -inst SIMTEST -det_type FLAT -dbname @DBNAME@ -filter i -time_begin 2008/1/1 -time_end 2008/1/2" +### launch STR "chiptool -dbname @DBNAME@ -set_label proc -label wait -dbname @DBNAME@ -time_begin 2008/1/1 -time_end 2008/1/2" +### END + +### XXX still not quite there.... +### the 'checks' or maybe the 'launch' needs to include a '-n_exp_min' ? +### Index: /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.basic.auto =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.basic.auto (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.basic.auto (revision 22161) @@ -0,0 +1,119 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis + +automate METADATA + name STR BIAS + check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" + ncheck S32 20 + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@" +END + +automate METADATA + name STR DARK + check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -dbname @DBNAME@" +END + +automate METADATA + name STR SHUTTER + check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-r + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type FLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-i + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type FLAT -select_filter i -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter i -dbname @DBNAME@" +END + +automate METADATA + name STR OBJECT-i + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter i -det_type FLAT" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc" + block STR NONE +END + +automate METADATA + name STR OBJECT-r + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLAT" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter r -label wait -set_label proc" + block STR NONE +END + +### Stack automation??? +automate METADATA + name STR STACK + regular STR "stacktool -definebyquery -all -workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@" +END + +### Diff automation??? +automate METADATA + name STR DIFF + regular STR "difftool -definebyquery -workdir file://@CWD@/diff -good_frac 0.2 -dbname @DBNAME@" +END + +### Magic automation??? +automate METADATA + name STR MAGIC + regular STR "magictool -definebyquery -workdir file://@CWD@/magic -good_frac 0.2 -dbname @DBNAME@" +END + + + + +### automate METADATA +### name STR OBJECT-r +### check STR "detselect -search -inst @CAMERA@ -det_type FLAT -dbname @DBNAME@ -filter r" +### launch STR "chiptool -unblock -label simtest-r -dbname @DBNAME@" +### block STR "chiptool -unmasked -label simtest-r -dbname @DBNAME@" +### END + +### automate METADATA +### name STR OBJECT-i +### check STR "detselect -search -inst @CAMERA@ -det_type FLAT -dbname @DBNAME@ -filter i" +### launch STR "chiptool -unblock -label simtest-i -dbname @DBNAME@" +### block STR "chiptool -unmasked -label simtest-i -dbname @DBNAME@" +### END + +### there is a weakness in the label / block business: the labels are +### generic, and the blocks are against those fairly generic words. +### that makes it difficult to block and unblock science exposures +### based on different detrend types that are available. For example, +### to block the r-band against the absence of the desired r-band +### flat, we would need to specify a label specific to the r-band +### images and remove that block when ready. + +### a better approach might be to modify the labels rather than the +### blocks. we can define, eg, chiptool -set_label -definebyquery to +### turn assign the label names based on various properties of the +### interesting images. + +### here is what the automate element might look like for such a circumstance: + +### automate METADATA +### name STR OBJECT-i +### block STR "chiptool -dbname @DBNAME@ -unmasked -label wait -filter i -time_begin 2008/1/1 -time_end 2008/1/2" +### check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type FLAT -dbname @DBNAME@ -filter i -time_begin 2008/1/1 -time_end 2008/1/2" +### launch STR "chiptool -dbname @DBNAME@ -set_label proc -label wait -dbname @DBNAME@ -time_begin 2008/1/1 -time_end 2008/1/2" +### END + +### XXX still not quite there.... +### the 'checks' or maybe the 'launch' needs to include a '-n_exp_min' ? +### Index: /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.basic.config =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.basic.config (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.basic.config (revision 22161) @@ -0,0 +1,112 @@ + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR SIMTEST + NIMAGES S32 20 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR SIMTEST + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 10, 10 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,i + @EXPTIMES F32 0.1,20.0,20.0 + + NSETUP S32 5 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 3 + DITHER.ND S32 2 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,i + @EXPTIMES F32 10.00,10.00 + @SKYMAGS F32 20.86,20.15 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + + +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 1 + DITHER.ND S32 1 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,r,i,i + @EXPTIMES F32 5.0, 240.0, 5.0, 240.0 + @SKYMAGS F32 20.86,20.86,20.15,20.15 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + Index: /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.det =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.det (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.det (revision 22161) @@ -0,0 +1,42 @@ + +detRunDef MULTI + +detRunDef METADATA + detRunName STR BIAS + prereq STR NONE + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type BIAS -select_exp_type BIAS" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR DARK + prereq STR BIAS + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type DARK -select_exp_type DARK" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR SHUTTER + prereq STR DARK + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-r + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-i + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter i -select_exp_type FLAT -select_filter i" + dbname STR eamtest + pantaskState STR NULL +END Index: /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.detverify.auto =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.detverify.auto (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.detverify.auto (revision 22161) @@ -0,0 +1,78 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +automate METADATA + name STR BIAS + check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" + ncheck S32 20 + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -mode master -dbname @DBNAME@" +END + +automate METADATA + name STR BIAS_VERIFY + check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -mode verify -dbname @DBNAME@" +END + +automate METADATA + name STR DARK + check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -mode master -dbname @DBNAME@" +END + +automate METADATA + name STR DARK_VERIFY + check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -mode verify -dbname @DBNAME@" +END + +automate METADATA + name STR SHUTTER + check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type SHUTTER -mode master -dbname @DBNAME@" +END + +automate METADATA + name STR SHUTTER_VERIFY + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -mode verify -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-r + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type FLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -mode master -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-i + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type FLAT -select_filter i -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter i -mode master -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT_VERIFY-r + check STR "detselect -search -inst @CAMERA@ -det_type FLAT -filter r -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type FLAT -select_filter r -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -mode verify -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT_VERIFY-i + check STR "detselect -search -inst @CAMERA@ -det_type FLAT -filter i -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type FLAT -select_filter i -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter i -mode verify -dbname @DBNAME@" +END + Index: /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.detverify.config =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.detverify.config (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.detverify.config (revision 22161) @@ -0,0 +1,37 @@ + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR SIMTEST + NIMAGES S32 20 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR SIMTEST + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 10, 10 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,i + @EXPTIMES F32 0.1,20.0,20.0 + + NSETUP S32 5 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + Index: /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.flatcorr.auto =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.flatcorr.auto (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.flatcorr.auto (revision 22161) @@ -0,0 +1,77 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +# this is a simplified automate script which only models BIAS and FLAT. +# generate appropriate simulated data with the FLATCORR recipe + +# wait until 20 bias frames are registered, then define a new bias run +automate METADATA + name STR BIAS + check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" + ncheck S32 20 + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@" +END + +# wait until the master bias frame is registered, then define a new dark run +automate METADATA + name STR DARK + check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -dbname @DBNAME@" +END + +# wait until the master dark frame is registered, then define a new shutter run +automate METADATA + name STR SHUTTER + check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@" +END + +# wait until the master bias frame is registered, then define a new flat-r run +automate METADATA + name STR RAWFLAT-r + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT_RAW -filter r -select_exp_type FLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT_RAW -filter r -dbname @DBNAME@" +END + +# wait until the master bias frame is registered, then define a new flat-i run +automate METADATA + name STR RAWFLAT-i + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT_RAW -filter i -select_exp_type FLAT -select_filter i -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT_RAW -filter i -dbname @DBNAME@" +END + +## # wait until the master flat-r frame is registered, then unblock the r-band data for chip-level processing +## automate METADATA +## name STR OBJECT-i +## check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter i -det_type FLAT" +## launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc" +## block STR NONE +## END +## +## # wait until the master flat-i frame is registered, then unblock the i-band data for chip-level processing +## automate METADATA +## name STR OBJECT-r +## check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLAT" +## launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter r -label wait -set_label proc" +## block STR NONE +## END +## +## # wait until the r-band is done with camera, then define a new flat-correction run +## automate METADATA +## name STR FLATCORR-i +## check STR "camtool -dbname @DBNAME@ -search -inst @CAMERA@ -filter r" +## ncheck S32 8 +## launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc" +## block STR NONE +## END + +## flatcorr -definebyquery -dbname eamtest -filter r -set_workdir /data/alala.0/eugene/swtests/simtest/test.f1/work -set_reduction FLATCORR -set_dvodb /data/alala.0/eugene/swtests/simtest/test.f1/catdir.flatcorr -set_region 270.6,270.9:-23.8,-23.5 -set_filter r Index: /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.flatcorr.config =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.flatcorr.config (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.flatcorr.config (revision 22161) @@ -0,0 +1,189 @@ + +# generate a simple fake data set with 2x2 dithers at offset of +# 50, 100, 200 arcsec (camera is 530 arcsec on a side) + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR SIMTEST + NIMAGES S32 20 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR SIMTEST + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 10, 10 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,i + @EXPTIMES F32 0.1,20.0,20.0 + + NSETUP S32 5 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + +# a single image at the field center +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + + ### Carina (modestly high density) + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 + + # the simtest camera is 2k x 2k, 530 arcsec on a side + # XXX note that we cannot define the offsets in terms of the camera dimensions... + + OFFSET.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 1 + DITHER.ND S32 1 + + # XXX is DVODB used? + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,i + @EXPTIMES F32 10.00,10.00 + @SKYMAGS F32 20.86,20.15 + + # modest IQ range here + IQ_MIN F32 0.70 + IQ_MAX F32 0.90 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + +# dither set @ scale 1 = 60 arcsec +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + + ### Carina (modestly high density) + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 + + # XXX note that we cannot define the offsets in terms of the camera dimensions... + OFFSET.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 2 + DITHER.ND S32 2 + + # XXX is DVODB used? + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,i + @EXPTIMES F32 10.00,10.00 + @SKYMAGS F32 20.86,20.15 + + # modest IQ range here + IQ_MIN F32 0.70 + IQ_MAX F32 0.90 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + +# dither set @ scale 2 = 120 arcsec +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + + ### Carina (modestly high density) + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 + + # XXX note that we cannot define the offsets in terms of the camera dimensions... + OFFSET.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 120.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 120.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 2 + DITHER.ND S32 2 + + # XXX is DVODB used? + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,i + @EXPTIMES F32 10.00,10.00 + @SKYMAGS F32 20.86,20.15 + + # modest IQ range here + IQ_MIN F32 0.70 + IQ_MAX F32 0.90 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + +# dither set @ scale 3 = 240 arcsec +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + + ### Carina (modestly high density) + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 + + # XXX note that we cannot define the offsets in terms of the camera dimensions... + OFFSET.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 240.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 240.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 2 + DITHER.ND S32 2 + + # XXX is DVODB used? + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,i + @EXPTIMES F32 10.00,10.00 + @SKYMAGS F32 20.86,20.15 + + # modest IQ range here + IQ_MIN F32 0.70 + IQ_MAX F32 0.90 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END Index: /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.megacam.config =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.megacam.config (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.megacam.config (revision 22161) @@ -0,0 +1,67 @@ + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR MEGACAM + + BIAS.LEVEL F32 100.0 + BIAS.RANGE F32 10.0 + + NIMAGE S32 15 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR MEGACAM + + DARK.MIN F32 100.0 + DARK.MAX F32 200.0 + + NIMAGE S32 15 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR MEGACAM + + FILTERS STR g,r,i + @EXPTIMES F32 60.0,30.0,30.0 + + NSETUP S32 15 +END + +SEQUENCE_SKIP METADATA + OBSTYPE STR OBJECT + CAMERA STR MEGACAM + + CENTER.RA F32 10.0 + CENTER.DEC F32 60.0 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 5 + OFFSET.ND S32 5 + + DITHER.RA F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 2 + DITHER.ND S32 2 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.grizy + FILTERS STR g,r,i + @EXPTIMES F32 60.0,30.0,30.0 + @SKYMAGS F32 21.0,20.5,20.0 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.5 + IQ_MAX F32 0.8 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + Index: /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/simtest.pro (revision 22161) @@ -0,0 +1,133 @@ +## simtest.pro : automatic full-suite simulated IPP test : -*- sh -*- + +## there are four features which define a simtest run: +## the camera : any camera can be used, the sim... cameras are defined to be quick tests +## the sequence : this file defines the set of observations generated +## the auto : the automation file defines the sequence of analysis performed +## the recipe : the ppsim recipe defines the features which are included in the generate data + +## these are not completely independent: a certain sequence may be +## required to yield a sensible result for a specific auto, and a +## specific recipe may also be needed + +if ($?PPSIM_RECIPE == 0) + $PPSIM_RECIPE = default +end +if ($?SIMTEST_CAMERA == 0) + $SIMTEST_CAMERA = SIMTEST +end +if ($?SIMTEST_SEQUENCE == 0) + $SIMTEST_SEQUENCE = simtest.basic.config +end +if ($?SIMTEST_AUTO == 0) + $SIMTEST_AUTO = simtest.basic.auto +end +if ($?SIMTEST_THREADS == 0) + $SIMTEST_THREADS = 0 +end + +macro simtest + if ($0 != 4) + echo "USAGE: simtest (dbname) (hostname) (init)" + echo " if (init) == run : just run the analysis tasks" + echo " if (init) == inject : (re)create the database and (re)inject the images" + echo " if (init) == new : restart completely from scratch" + echo "" + echo "there are additional options which may be specified by setting the following variable names:" + echo " PPSIM_RECIPE : define the recipe to use when generating the fake data" + echo " SIMTEST_CAMERA : define an alternate camera (otherwise uses entry in sequence file)" + echo " SIMTEST_SEQUENCE : define the set of observations generated (simtest.basic.config)" + echo " SIMTEST_AUTO : define the analysis steps to perform (simtest.auto)" + echo " SIMTEST_THREADS : set the number of threads for the processing node (0)" + echo "" + echo "the following macros can be used to set up specific simtest suites:" + echo " simtest.setup.basic : run standard simtest suite" + echo " simtest.setup.detverify : run detrend creation and detrend verification" + echo " simtest.setup.flatcorr : run a flat-field correction demonstration" + break + end + + $dbname = $1 + $hostname = $2 + $init = $3 + if (("$init" != "run") && ("$init" != "inject") && ("$init" != "new")) + echo "USAGE: simtest (dbname) (hostname) (init)" + echo " if (init) == run : just run the analysis tasks" + echo " if (init) == inject : (re)create the database and (re)inject the images" + echo " if (init) == new : restart completely from scratch" + break + end + + if (("$init" == "new") || ("$init" == "inject")) + # XXX this will fail and exit the script if the db does not exist or is old... + exec pxadmin -delete -dbname $dbname + + # XXX this gives warnings if the db exists... + exec pxadmin -create -dbname $dbname + + if ("$init" == "new") + # the labels "wait" and "proc" are special names used in automate.pro + + $ppsim = "ppSimSequence $MODULES:0/$SIMTEST_SEQUENCE simtest.mkimages simtest.inject -path raw -workdir work -dbname $dbname -label wait -dvodb DVO -tess_id TESS" + if ("$PPSIM_RECIPE" != "default") + $ppsim = $ppsim -ppsim_recipe $PPSIM_RECIPE + end + $ppsim = $ppsim -camera $SIMTEST_CAMERA + + exec $ppsim + + exec source simtest.mkimages + + ## XXX this will need to be optional as well + file TESS/Images.dat found + if ($found == 0) + exec skycells 8 -scale 0.2 -D CATDIR TESS + end + end + + exec chiptool -dbname $dbname -block -label wait + exec source simtest.inject + end + + module pantasks.pro + module automate.pro + + module.tasks + + if ($SIMTEST_THREADS == 0) + controller host add $hostname + else + controller host add $hostname -threads $SIMTEST_THREADS + end + + add.database $dbname + + automate.load $SIMTEST_AUTO $SIMTEST_CAMERA $dbname + run +end + +# XXX add a test function to pxadmin to check for db existence + +## currently defined options for sequence, etc: + +# sequence: simtest.basic.config, simtest.flatcorr.config +# auto: simtest.basic.auto, simtest.flatcorr.auto + +macro simtest.setup.basic + $PPSIM_RECIPE = default + $SIMTEST_SEQUENCE = simtest.basic.config + $SIMTEST_AUTO = simtest.basic.auto +end + +macro simtest.setup.detverify + $PPSIM_RECIPE = default + $SIMTEST_SEQUENCE = simtest.detverify.config + $SIMTEST_AUTO = simtest.detverify.auto +end + +macro simtest.setup.flatcorr + $PPSIM_RECIPE = FLATCORR + $SIMTEST_SEQUENCE = simtest.flatcorr.config + $SIMTEST_AUTO = simtest.flatcorr.auto +end + Index: /branches/eam_branches/eam_branch_20080918/ippTasks/site.manoa.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/site.manoa.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/site.manoa.pro (revision 22161) @@ -0,0 +1,52 @@ +## site.mhpcc.pro : example configuration script for the MHPCC IPP cluster : -*- sh -*- + +## a site configuration needs to define the following pieces of information: +## 1) the machines used for processing +## 2) the default_host and workdir_template variables +## 3) the ipphost table + +macro init.cluster.po + $PARALLEL = 1 + controller exit true + + # po05 -- using for pantasks + # po23 -- broken perl + + controller host add po02 + controller host add po03 + controller host add po04 +# controller host add po05 + controller host add po06 + controller host add po07 +# controller host add po08 +# controller host add po09 +# controller host add po10 +# controller host add po11 +# controller host add po12 +# controller host add po13 +# controller host add po14 +# controller host add po15 +# controller host add po16 +# controller host add po17 +# controller host add po18 +# controller host add po19 +# controller host add po20 +# controller host add po21 +# controller host add po22 +# controller host add po23 +# controller host add po24 +end + +macro init.cluster.sn + $PARALLEL = 1 + controller exit true + controller host add sn2 + controller host add sn3 + controller host add sn4 + controller host add sn5 +end + +macro init.site + queueload tmp -x "cat $MODULES:0/ipphosts.manoa.config" + ipptool2book tmp ipphosts -key camera +end Index: /branches/eam_branches/eam_branch_20080918/ippTasks/site.mhpcc.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/site.mhpcc.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/site.mhpcc.pro (revision 22161) @@ -0,0 +1,70 @@ +## site.mhpcc.pro : example configuration script for the MHPCC IPP cluster : -*- sh -*- + +## a site configuration needs to define the following pieces of information: +## 1) the machines used for processing +## 2) the default_host and workdir_template variables +## 3) the ipphost table + +macro init.cluster.mhpcc + $PARALLEL = 1 + controller exit true +# controller host add ipp005 + controller host add ipp006 + controller host add ipp007 + controller host add ipp008 + controller host add ipp009 + controller host add ipp010 + controller host add ipp011 +# controller host add ipp016 +# controller host add ipp017 + controller host add ipp018 + controller host add ipp020 + controller host add ipp021 +end + +## override the basic inits set in pantasks.pro +macro init.copy.mhpcc + if ($0 != 2) + echo "USAGE: init.copy.mhpcc (nebulous)" + echo "nebulous may be 'on' or 'off'" + break + end + + if (("$1" != "on") && ("$1" != "off")) + echo "USAGE: init.copy.mhpcc (nebulous)" + echo "nebulous may be 'on' or 'off'" + break + end + + # XXX this is only used by summit.copy.pro. move this as a check into summit.copy.pro? + $COMPRESS = 1 + + # the templates are used if we have a class_id/host relationship; + # if none is found, the default values are used + # XXX not sure how to handle the .N value if we need to use more than one + + if ("$1" == "on") + $NEBULOUS = 1 + $default_host = ipp004 + $workdir_template = neb://@HOST@.0 + else + $NEBULOUS = 0 + $default_host = ipp004 + $workdir_template = /data/@HOST@.0 + end +end + +macro init.site + init.cluster.mhpcc + init.copy.mhpcc on + + queueload tmp -x "cat $MODULES:0/ipphosts.mhpcc.config" + ipptool2book tmp ipphosts -key camera +end + +macro init.site.nohosts + init.copy.mhpcc on + + queueload tmp -x "cat $MODULES:0/ipphosts.mhpcc.config" + ipptool2book tmp ipphosts -key camera +end Index: /branches/eam_branches/eam_branch_20080918/ippTasks/stack.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/stack.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/stack.pro (revision 22161) @@ -0,0 +1,289 @@ +## stack.pro : tasks for image stacking : -*- sh -*- + +## This file contains panTasks definitions for performing the image stacking. +## After a stack (with associated stack_id) is defined, the stack is performed +## (tasks in stackSumSkyfile). + +# test for required global variables +check.globals + +### Initialise the books containing the tasks to do +book init stackSumSkyfile +book init stackCleanup + +### Database lists +$stackSkycell_DB = 0 +$stackCleanup_DB = 0 + +### Check status of stacking tasks +macro stack.status + book listbook stackSumSkyfile + book listbook stackCleanup +end + +### Reset stacking tasks +macro stack.reset + book init stackSumSkyfile + book init stackCleanup +end + +### Turn stacking tasks on +macro stack.on + task stack.skycell.load + active true + end + task stack.skycell.run + active true + end +end + +### Turn stacking tasks off +macro stack.off + task stack.skycell.load + active false + end + task stack.skycell.run + active false + end +end + +macro stack.cleanup.on + task stack.cleanup.load + active true + end + task stack.cleanup.run + active true + end +end + +macro stack.cleanup.off + task stack.cleanup.load + active false + end + task stack.cleanup.run + active false + end +end + + +### Load tasks for doing the stack +### Tasks are loaded into stackSumSkyfile. +task stack.skycell.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/stack.skycell.log + + task.exec + $run = stacktool -tosum + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$stackSkycell_DB + $run = $run -dbname $DB:$stackSkycell_DB + $stackSkycell_DB ++ + if ($stackSkycell_DB >= $DB:n) set stackSkycell_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout stackSumSkyfile -key stack_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook stackSumSkyfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup stackSumSkyfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the stack overlaps +### Tasks are taken from stackSumSkyfile. +task stack.skycell.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + book npages stackSumSkyfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in stackSumSkyfile (pantaskState == INIT) + book getpage stackSumSkyfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword stackSumSkyfile $pageName pantaskState RUN + book getword stackSumSkyfile $pageName stack_id -var STACK_ID + book getword stackSumSkyfile $pageName tess_id -var TESS_DIR + book getword stackSumSkyfile $pageName skycell_id -var SKYCELL_ID + book getword stackSumSkyfile $pageName workdir -var WORKDIR_TEMPLATE + book getword stackSumSkyfile $pageName dbname -var DBNAME + book getword stackSumSkyfile $pageName state -var RUN_STATE + + # set the host and workdir based on the skycell hash + set.host.for.skycell $SKYCELL_ID + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + # XXX old code: + # host anyhost + # $WORKDIR = $WORKDIR_TEMPLATE + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.stk.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $STACK_ID + + stdout $LOGDIR/stack.skycell.log + stderr $LOGDIR/stack.skycell.log + + $run = stack_skycell.pl --threads @MAX_THREADS@ --stack_id $STACK_ID --outroot $outroot --redirect-output --run-state $RUN_STATE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit stackSumSkyfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword stackSumSkyfile $options:0 pantaskState TIMEOUT + end +end + + +# select images ready for stack analysis +# new entries are added to stackCleanup +# skip already-present entries +task stack.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/stack.cleanup.log + + task.exec + $run = stacktool -pendingcleanuprun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$stackCleanup_DB + $run = $run -dbname $DB:$stackCleanup_DB + $stackCleanup_DB ++ + if ($stackCleanup_DB >= $DB:n) set stackCleanup_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout stackCleanup -key stack_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook stackCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup stackCleanup + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task stack.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + book npages stackCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in stackCleanup (pantaskState == INIT) + book getpage stackCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword stackCleanup $pageName pantaskState RUN + book getword stackCleanup $pageName camera -var CAMERA + book getword stackCleanup $pageName state -var CLEANUP_MODE + book getword stackCleanup $pageName stack_id -var STACK_ID + book getword stackCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and stack (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/stack.cleanup.log + stderr $LOGDIR/stack.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage stack --stage_id $STACK_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit stackCleanup $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword stackCleanup $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080918/ippTasks/summit.copy.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/summit.copy.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/summit.copy.pro (revision 22161) @@ -0,0 +1,474 @@ +## summit.copy.pro : tasks for the summit to IPP download : -*- sh -*- +## PanTasks scripts for Summit Copy + +## XXX note that this currently works with a single database as defined in .ipprc +## XXX tie the database for output to the database from which the datastore was determined + +# pztool -adddatastore -inst isp -telescope ps1 -uri http://otis1.ifa.hawaii.edu/ds/skyprobe/index.txt +# pztool -adddatastore -inst gpc1 -telescope ps1 -uri http://conductor/ds/gpc1/index.txt +# pztool -adddatastore -inst allskycam -telescope ps1 -uri http://otis1.ifa.hawaii.edu/ds/allskycam/index.txt + +# NOTE: workdir / volume mangling and nebulous. these tasks copy the +# imfiles from the summit, placing the output files in a directory +# which is based on the chip/host relationship. If nebulous is being +# used, the volume name is set based on the host; if nebulous is not +# being used, the workdir is set to include the host name. The copy +# operation is targetted to the same host by pcontrol. The value of +# workdir is set to be a template into which the appropriate value of +# @HOST@ may be substituted in later scripts. + +# test for required global variables +check.globals + +# list of DataStores to pull data from +book init pzDataStore +# list of summit exps that need to be queried +book init pzPendingExp +# list of summit imfiles that need to be downloaded +book init pzPendingImfile + +macro copy.on + task pztool.datastore + active true + end + task pzgetexp + active true + end + task pztool.pendingexp + active true + end + task pzgetimfile + active true + end + task pztool.pendingimfile + active true + end + task summit_copy + active true + end + task pztool.clearfault + active true + end +end + +macro copy.off + task pztool.datastore + active false + end + task pzgetexp + active false + end + task pztool.pendingexp + active false + end + task pzgetimfile + active false + end + task pztool.pendingimfile + active false + end + task summit_copy + active false + end + task pztool.clearfault + active false + end +end + +# these variables will cycle through the known database names +$pztoolDatastore_DB = 0 +$pztoolPendingExp_DB = 0 +$pztoolPendingImfile_DB = 0 +$pztoolClearFault_DB = 0 + +# build a book of datastores to poll for data +task pztool.datastore + host local + + # timeout shorter than exec so jobs do not build up + periods -exec 10 + periods -poll 1 + periods -timeout 20 + npending 1 + # trange 16:00 23:59 + # trange 00:00 04:00 + + task.exec + if ($DB:n == 0) + option DEFAULT + command pztool -datastore + else + # save the DB name for the exit tasks + option $DB:$pztoolDatastore_DB + command pztool -datastore -dbname $DB:$pztoolDatastore_DB + $pztoolDatastore_DB ++ + if ($pztoolDatastore_DB >= $DB:n) set pztoolDatastore_DB = 0 + end + end + + # success + task.exit 0 + # flush pzDataStore book + book init pzDataStore + # convert 'stdout' to book format + ipptool2book stdout pzDataStore -key camera:telescope -uniq -setword dbname $options:0 + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout + end +end + +$datastore_index = 0 + +# run pzgetexp periodically to populate pzPendingExp in the database (no I/O) +# this task is querying the data store for a list of exposures ("filesets") +# and inserting these into a db table on the local cluster (pzPendingExp) +task pzgetexp + periods -exec 60 + periods -poll 1 + periods -timeout 700 + # trage 16:00 23:59 + # trage 00:00 04:00 + npending 1 + host local + + task.exec + # find an exp that needs imfiles fetched + book getpage pzDataStore $datastore_index -var pageName + if ("$pageName" == "NULL") break + + # increment our pzDataStore index and loop back to 0 and the end of the + # book + $datastore_index ++ + book npages pzDataStore -var npages + if ($datastore_index == $npages ) + $datastore_index = 0 + end + + book getword pzDataStore $pageName camera -var CAMERA + book getword pzDataStore $pageName telescope -var TELESCOPE + book getword pzDataStore $pageName uri -var URI + book getword pzDataStore $pageName dbname -var DBNAME + + # store the current page + options $pageName + + $run = pzgetexp -uri $URI -inst $CAMERA -telescope $TELESCOPE -dbname $DBNAME -timeout 650 + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + + command $run + end + + task.exit 0 + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout + end +end + +# build a book of exps/filesetids that need to be queried +task pztool.pendingexp + host local + + periods -exec 10 + periods -poll 1 + periods -timeout 20 + # trange 16:00 23:59 + # trange 00:00 04:00 + npending 1 + + task.exec + if ($DB:n == 0) + option DEFAULT + command pztool -pendingexp -limit 5 + else + # save the DB name for the exit tasks + option $DB:$pztoolPendingExp_DB + command pztool -pendingexp -limit 5 -dbname $DB:$pztoolPendingExp_DB + $pztoolPendingExp_DB ++ + if ($pztoolPendingExp_DB >= $DB:n) set pztoolPendingExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout pzPendingExp -key exp_name:camera:telescope -uniq -setword dbname $options:0 -setword pantaskState INIT + + # delete existing entries in the appropriate pantaskStates + process_cleanup pzPendingExp + end + + task.exit default + showcommand failure + if ($VERBOSE) + echo "*** stdout ***" + queueprint stdout + echo "*** stderr ***" + queueprint stderr + end + end + task.exit timeout + showcommand timeout + end +end + +# run pzgetimfiles on pending exps. analogous to pzgetexp, this task +# is downloading a list of imfiles reported by the datastore for a +# given exposure ("fileset"), and placing the result in the local +# database table of imfiles +task pzgetimfile + periods -exec 0.05 + periods -poll 0.025 + periods -timeout 700 + # trage 16:00 23:59 + # trage 00:00 04:00 + host local + npending 10 + + task.exec + if ($NETWORK == 0) break + + # if we are waiting on data, make the interval long + book npages pzPendingExp -var N + if ($N == 0) + periods -exec 20 + break + end + periods -exec 0.05 + + # find an exp that needs imfiles fetched + book getpage pzPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # set that exp to run + book setword pzPendingExp $pageName pantaskState RUN + + book getword pzPendingExp $pageName exp_name -var EXP_NAME + book getword pzPendingExp $pageName camera -var CAMERA + book getword pzPendingExp $pageName telescope -var TELESCOPE + book getword pzPendingExp $pageName dateobs -var DATEOBS + book getword pzPendingExp $pageName exp_type -var EXP_TYPE + book getword pzPendingExp $pageName uri -var URI + book getword pzPendingExp $pageName imfiles -var IMFILES + book getword pzPendingExp $pageName dbname -var DBNAME + + # store the current page + options $pageName + + $batman = $EXP_NAME + # Sidik says we should use a longer timeout + $run = pzgetimfiles -uri $URI -filesetid $batman -inst $CAMERA -telescope $TELESCOPE -dbname $DBNAME -timeout 650 + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit 0 + process_exit pzPendingExp $options:0 $JOB_STATUS + end + + task.exit default + showcommand failure + process_exit pzPendingExp $options:0 $JOB_STATUS + end + + task.exit timeout + showcommand timeout + book setword pzPendingExp $options:0 pantaskState TIMEOUT + end +end + + +# build a book of imfiles/files that need to be downloaded +task pztool.pendingimfile + host local + + periods -exec 30 + periods -poll 1 + periods -timeout 120 + # trage 16:00 23:59 + # trage 00:00 04:00 + npending 1 + + # select entries from the current DB; cycle to the next DB, if it exists + # iff the DB list is not set, use the value defined in .ipprc + task.exec + if ($DB:n == 0) + option DEFAULT + command pztool -pendingimfile -limit 40 + else + # save the DB name for the exit tasks + option $DB:$pztoolPendingImfile_DB + command pztool -pendingimfile -limit 60 -dbname $DB:$pztoolPendingImfile_DB + $pztoolPendingImfile_DB ++ + if ($pztoolPendingImfile_DB >= $DB:n) set pztoolPendingImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout pzPendingImfile -key exp_name:camera:telescope:class:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + book shuffle pzPendingImfile + + # delete existing entries in the appropriate pantaskStates + process_cleanup pzPendingImfile + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout + end +end + +# retreive an imfile with dsget and then call pztool -copydone +task summit_copy + periods -exec 5 + periods -poll 0.05 + periods -timeout 650 + # trage 16:00 23:59 + # trage 00:00 04:00 + + task.exec + if ($NETWORK == 0) break + + # if we are waiting on data, make the interval long + book npages pzPendingImfile -var N + if ($N == 0) + periods -exec 20 + break + end + periods -exec 0.05 + + # find an exp that needs imfiles fetched + book getpage pzPendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # set that exp to run + book setword pzPendingImfile $pageName pantaskState RUN + + book getword pzPendingImfile $pageName uri -var URI + book getword pzPendingImfile $pageName bytes -var BYTES + book getword pzPendingImfile $pageName md5sum -var MD5SUM + book getword pzPendingImfile $pageName dateobs -var DATEOBS + book getword pzPendingImfile $pageName exp_name -var EXP_NAME + book getword pzPendingImfile $pageName camera -var CAMERA + book getword pzPendingImfile $pageName telescope -var TELESCOPE + book getword pzPendingImfile $pageName class -var CLASS + book getword pzPendingImfile $pageName class_id -var CLASS_ID + book getword pzPendingImfile $pageName dbname -var DBNAME + + set.host.for.camera $CAMERA $CLASS_ID + + # 2007-08-30T05:09:59Z + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 5 2 MONTH + substr $DATEOBS 8 2 DAY + + # we need to set the workdir based on 1) nebulous or not? 2) chip/host relationship + # this function uses workdir_template, default_host, volume_template, volume_default, + # it sets workdir and volume + set.workdir.by.camera $CAMERA $CLASS_ID $workdir_template $default_host workdir_base + + # figure out filename + # XXX may need to use sprintf here + $FILENAME = $workdir_base/$CAMERA/$YEAR\$MONTH\$DAY/$EXP_NAME/$EXP_NAME.$CLASS_ID.fits + $workdir = $workdir_template/$CAMERA/$YEAR\$MONTH\$DAY + + # workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb://@HOST@.0/gpc1/20080130 + + # filename examples: + # file://data/ipp005.0/gpc1/20080130/o4437g0025d/o4437g0025d.XY05.fits + # neb://@HOST@.0/gpc1/20080130/o4437g0025d/o4437g0025d.XY05.fits + + book setword pzPendingImfile $pageName filename $FILENAME + + $run = summit_copy.pl --uri $URI --filename $FILENAME --exp_name $EXP_NAME --inst $CAMERA --telescope $TELESCOPE --class $CLASS --class_id $CLASS_ID --bytes $BYTES --md5 $MD5SUM --end_stage reg --workdir $workdir --dbname $DBNAME --timeout 120 --verbose + if ($COMPRESS) + $run = $run --compress + else + $run = $run --bytes $BYTES + end + if (("$MD5SUM" != "NULL") && ("$MD5SUM" != "0") && (not($COMPRESS))) + $run = $run --md5 $MD5SUM + end + if ($NEBULOUS) + $run = $run --nebulous + end + # add_standard_args run + + # store the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit pzPendingImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword pzPendingImfile $options:0 pantaskState TIMEOUT + end +end + +task pztool.clearfault + host local + + # -exec is set much longer the first time this task runs + periods -exec 7 + periods -poll 1 + periods -timeout 30 + npending 1 + + task.exec + if ($DB:n == 0) + command pztool -clearcommonfaults + else + command pztool -clearcommonfaults -dbname $DB:$pztoolClearFault_DB + + $pztoolClearFault_DB ++ + if ($pztoolClearFault_DB >= $DB:n) set pztoolClearFault_DB = 0 + end + periods -exec 1800 + end + + # success + task.exit 0 + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout + end +end Index: /branches/eam_branches/eam_branch_20080918/ippTasks/warp.pro =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTasks/warp.pro (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTasks/warp.pro (revision 22161) @@ -0,0 +1,416 @@ +## warp.pro : image warping tasks : -*- sh -*- +## This file contains panTasks definitions for performing the image warping. + +### This is done in two (main) steps. After a warp (with associated warp_id) is defined, +### overlaps between the exposure being warped and skycells are calculated (tasks in warpInputExp). +### Then for each skycell, the warp is made (tasks in warpPendingSkycell). + +### Setups +check.globals + +### Initialise the books containing the tasks to do +book init warpInputExp +book init warpPendingSkyCell +book init warpPendingCleanup + +### Database lists +$warpExp_DB = 0 +$warpSkycell_DB = 0 +$warpCleanup_DB = 0 + +### Check status of warping tasks +macro warp.status + book listbook warpInputExp + book listbook warpPendingSkyCell + book listbook warpPendingCleanup +end + +### Reset warping tasks +macro warp.reset + book init warpInputExp + book init warpPendingSkyCell + book init warpPendingCleanup +end + +### Turn warping tasks on +macro warp.on + task warp.exp.load + active true + end + task warp.exp.run + active true + end + task warp.skycell.load + active true + end + task warp.skycell.run + active true + end +end + +### Turn warping tasks off +macro warp.off + task warp.exp.load + active false + end + task warp.exp.run + active false + end + task warp.skycell.load + active false + end + task warp.skycell.run + active false + end +end + +macro warp.cleanup.on + task warp.cleanup.load + active true + end + task warp.cleanup.run + active true + end +end +macro warp.cleanup.off + task warp.cleanup.load + active false + end + task warp.cleanup.run + active false + end +end + +### Load tasks for calculating the warp overlaps +### Tasks are loaded into warpInputExp. +task warp.exp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/warp.exp.log + + task.exec + $run = warptool -tooverlap + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$warpExp_DB + $run = $run -dbname $DB:$warpExp_DB + $warpExp_DB ++ + if ($warpExp_DB >= $DB:n) set warpExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout warpInputExp -key warp_id:fake_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpInputExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpInputExp + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the warp overlaps +### Tasks are taken from warpInputExp. +task warp.exp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages warpInputExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in warpInputExp (pantaskState == INIT) + book getpage warpInputExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpInputExp $pageName pantaskState RUN + book getword warpInputExp $pageName warp_id -var WARP_ID + book getword warpInputExp $pageName camera -var CAMERA + book getword warpInputExp $pageName workdir -var WORKDIR_TEMPLATE + book getword warpInputExp $pageName dbname -var DBNAME + # XXX change tess_id to tess_dir when schema is changed + book getword warpInputExp $pageName tess_id -var TESS_DIR + book getword warpInputExp $pageName exp_tag -var EXP_TAG + + # set the host and workdir (default) + set.host.for.camera $CAMERA $WARP_ID + set.workdir.by.camera $CAMERA $WARP_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure + sprintf logfile "%s/%s/%s.wrp.%s.log" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID + + stdout $LOGDIR/warp.exp.log + stderr $LOGDIR/warp.exp.log + + # XXX warp_overlap.pl differs from the standard script : it does not have an 'outroot' argument, and it does not take '--redirect' + $run = warp_overlap.pl --warp_id $WARP_ID --camera $CAMERA --tess_dir $TESS_DIR --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit warpInputExp $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpInputExp $options:0 pantaskState TIMEOUT + end +end + + +### Load tasks for doing the warps +### Tasks are loaded into warpPendingSkyCell. +task warp.skycell.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/warp.skycell.log + + task.exec + $run = warptool -towarped + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$warpSkycell_DB + $run = $run -dbname $DB:$warpSkycell_DB + $warpSkycell_DB ++ + if ($warpSkycell_DB >= $DB:n) set warpSkycell_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + # XXX change tess_id to tess_dir when db is updated + ipptool2book stdout warpPendingSkyCell -key warp_id:skycell_id:tess_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpPendingSkyCell + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpPendingSkyCell + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the warp overlaps +### Tasks are taken from warpPendingSkyCell. +task warp.skycell.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages warpPendingSkyCell -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in warpPendingSkyCell (pantaskState == INIT) + book getpage warpPendingSkyCell 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpPendingSkyCell $pageName pantaskState RUN + book getword warpPendingSkyCell $pageName warp_id -var WARP_ID + book getword warpPendingSkyCell $pageName skycell_id -var SKYCELL_ID + book getword warpPendingSkyCell $pageName camera -var CAMERA + book getword warpPendingSkyCell $pageName workdir -var WORKDIR_TEMPLATE + book getword warpPendingSkyCell $pageName dbname -var DBNAME + # XXX change tess_id to tess_dir when schema is changed + book getword warpPendingSkyCell $pageName tess_id -var TESS_DIR + book getword warpPendingSkyCell $pageName exp_tag -var EXP_TAG + book getword warpPendingSkyCell $pageName state -var RUN_STATE + + # set the host and workdir based on the skycell hash + set.host.for.skycell $SKYCELL_ID + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure + sprintf outroot "%s/%s/%s.wrp.%s.%s" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID $SKYCELL_ID + + stdout $LOGDIR/warp.skycell.log + stderr $LOGDIR/warp.skycell.log + + $run = warp_skycell.pl --threads @MAX_THREADS@ --warp_id $WARP_ID --skycell_id $SKYCELL_ID --tess_dir $TESS_DIR --camera $CAMERA --outroot $outroot --redirect-output --run-state $RUN_STATE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit warpPendingSkyCell $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpPendingSkyCell $options:0 pantaskState TIMEOUT + end +end + +# select images ready for warp analysis +# new entries are added to warpPendingImfile +# skip already-present entries +task warp.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/warp.cleanup.log + + task.exec + $run = warptool -pendingcleanuprun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$warpCleanup_DB + $run = $run -dbname $DB:$warpCleanup_DB + $warpCleanup_DB ++ + if ($warpCleanup_DB >= $DB:n) set warpCleanup_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout warpPendingCleanup -key warp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpPendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpPendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task warp.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + book npages warpPendingCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in warpPendingCleanup (pantaskState == INIT) + book getpage warpPendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpPendingCleanup $pageName pantaskState RUN + book getword warpPendingCleanup $pageName camera -var CAMERA + book getword warpPendingCleanup $pageName state -var CLEANUP_MODE + book getword warpPendingCleanup $pageName warp_id -var WARP_ID + book getword warpPendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and warp (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/warp.cleanup.log + stderr $LOGDIR/warp.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit warpPendingCleanup $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpPendingCleanup $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080918/ippTools/.cvsignore =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/.cvsignore (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/.cvsignore (revision 22161) @@ -0,0 +1,16 @@ +Makefile +Makefile.in +aclocal.m4 +autom4te.cache +compile +config.guess +config.log +config.status +config.sub +configure +depcomp +install-sh +libtool +ltmain.sh +missing +pxtools.pc Index: /branches/eam_branches/eam_branch_20080918/ippTools/COPYING =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/COPYING (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/COPYING (revision 22161) @@ -0,0 +1,345 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. Index: /branches/eam_branches/eam_branch_20080918/ippTools/Makefile.am =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/Makefile.am (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/Makefile.am (revision 22161) @@ -0,0 +1,12 @@ +SUBDIRS = src share + +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA= pxtools.pc + +EXTRA_DIST = \ + pxtools.pc.in \ + autogen.sh + +CLEANFILES = *~ core core.* + +ACLOCAL_AMFLAGS = -I m4 Index: /branches/eam_branches/eam_branch_20080918/ippTools/TODO =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/TODO (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/TODO (revision 22161) @@ -0,0 +1,14 @@ +- change dettol -pretend -define... to return a metadata structure that shows what the created detRun would look like +- rename dettool -residdetrun -> -todetrunsummary +- add foreign key constraints to the database +- add the ability to remove dettool detResidExps +- change p4tool to accept input from/with magic masks +- change p5tool to accept input from warp * stack +- difftool/stacktool error handling +- move all large SQL statements out into separate files +- add the option of not automatically flowing through from reg -> chip -> cam + tools +- add workdir inheritance rooted in rawExp, eg. + select chipProcessedExp.workdir, rawExp.workdir, IFNULL(chipProcessedExp.workdir, rawExp.workdir) from chipProcessedExp join rawExp using(exp_tag); +- dettool/detRun support for filelevel +- combind dettool -definebytag & -definebyquery Index: /branches/eam_branches/eam_branch_20080918/ippTools/autogen.sh =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/autogen.sh (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/autogen.sh (revision 22161) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=pxtools +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +#($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $LIBTOOLIZE installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" +# DIE=1 +#} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +#($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $AUTOHEADER installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" +# DIE=1 +#} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +$LIBTOOLIZE --copy --force || echo "$LIBTOOLIZE failed" +$ACLOCAL -I m4 || echo "$ACLOCAL failed" +$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: /branches/eam_branches/eam_branch_20080918/ippTools/configure.ac =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/configure.ac (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/configure.ac (revision 22161) @@ -0,0 +1,64 @@ +AC_PREREQ(2.61) + +AC_INIT([ipptools], [1.1.36], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([autogen.sh]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_CONFIG_HEADER([src/config.h]) +AM_MAINTAINER_MODE + +IPP_STDCFLAGS + +AC_LANG(C) +AC_GNU_SOURCE +AC_PROG_CC_C99 +AC_PROG_INSTALL +AC_PROG_LIBTOOL + +PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0]) +PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) +PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.39]) + +PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}" +PXTOOLS_LIBS="${PSLIB_LIBS=} ${PSMODULES_LIBS=} ${IPPDB_LIBS=}" +AC_SUBST(PXTOOLS_CFLAGS,[$PXTOOLS_CFLAGS]) +AC_SUBST(PXTOOLS_LIBS,[$PXTOOLS_LIBS]) + +dnl check for PSDB support +TMP_CFLAGS=${CFLAGS} +CFLAGS="${CFLAGS=} ${PSLIB_CFLAGS}" + +AC_COMPILE_IFELSE([AC_LANG_PROGRAM( + [[#if !defined HAVE_PSDB + error: PSDB support is not avaiable + #endif + ]])], + [], + [AC_MSG_ERROR([psLib was built without PSDB support, bailing out.])] +) + +CFLAGS=${TMP_CFLAGS} + +dnl -- check for psParseErrorCodes -------------------------------------- + +AC_PATH_PROG([ERRORCODES], [psParseErrorCodes], [missing]) +if test "$ERRORCODES" = "missing" ; then + AC_MSG_ERROR([psParseErrorCodes is required]) +fi + +dnl check for perl +AC_PATH_PROG([PERL], [perl], [missing]) +if test "$PERL" = "missing" ; then + AC_MSG_ERROR([perl is required]) +fi + +IPP_STDOPTS +CFLAGS="${CFLAGS=} -Wall -Werror" + +AC_CONFIG_FILES([ + Makefile + src/Makefile + share/Makefile + pxtools.pc +]) +AC_OUTPUT Index: /branches/eam_branches/eam_branch_20080918/ippTools/doc/dettool.txt =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/doc/dettool.txt (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/doc/dettool.txt (revision 22161) @@ -0,0 +1,181 @@ +# returns a list of unassociated rawDetrendExp (rawDetrendExps that don't +# correspond to a exp_id in detInputExp) +dettool -pending -exp_type -inst .. + +# define a new detRun semi-automatically by selecting the component exps with +# the specified search criteria +dettool -definebyquery -det_type ... + +# create a detRun with the specified input exp_ids +dettool -definebyexp -det_type -exp_id 1 -exp_id 2 ... + +-- +# returns a list of rawImfiles that are associated with an detInputExp that do +# not appear in detProcessedImfiles +dettool -raw + +# add an imfile to detProcessedImfiles +dettool -addprocessed + +-- +# the list of class_ids that are ready to be stacked +# returns a list of det_id/iter/class_id/det_typs where class_id is unique per +# det_id (not a list of all exp_ids/class_ids) that do not have an entry in +# detStackedImfile +# results are masked by entries in detResidExpAnalysis +# may need to check detResidExp's even on iter 0 - ??? +dettool -tostack + +# list all of imfiles from different exps with the same class_id that have +# been processed +dettool -processed -wholestack -det_id ... + +# adds a stacked class_id and and optionally marks it for normalization +detool -addstacked [-pleasenormalize] -det_id -iter -class_id ... + +-- +# returns a list of det_ids/iters where the entire set of class_ids has been +# processed +dettool -tonormalize + +# returns the complete list of processed imfiles for the specified det_id +dettool -processed -unmask -det_id -iter + +# add per stacked imfile normalization statistics +# O: this could accept a MDC doc as input from stdin +dettool -addnormstat -det_id -iter -class_idi -norm F32 + +-- +# returns a list of det_ids/iter/class_id/uri from detNormStats. Stops +# returning entries after the class_is is inserted into detNormalizedImfile +dettool -normstat -det_id + +# inserts an entry into detNormalizedImfile +dettool -addnormalized -det_id -ter -class_id -uri + +-- +# returns a list of processed imfiles that have also been normalized with +# processed imfiles being masked out per detResidImfileAnalysis +# also returns stackedimfiles that have -pleasenormalize set to false +dettool -toresid -det_id -iter + +# inserts a per det_id/iter/exp_id/class_id residual +dettool -addresidimfile -det_id -exp_id -class_id -iter -stat F32 -stat_stdev F32 -uri -b1 -b2 + +# returns a list of det_id/iter/exp_ids but only for exp_ids that have resids +# for all of their imfiles +dettool -toresidexp + +# returns a list of residual imfile data +dettool -residimfile -det_id -iter -exp_id + +# inserts into detResidExpAnalysis +# sets the accept accept bool unless -reject is specified +dettool -addresidexp -det_id -iter -exp_id -jpeg1 -jpeg2 -stat F32 -stat_stdev +F32 [-reject] + +-- +# lists det_id/iter for detruns that have completed all residexps for their +# current iteration +dettool -residdetrun + +# lists all residexps for the given det_id/iter +dettool -residexp -det_id -iter + +# updates the given residexp to be accept/reject +dettool -updateresid -det_id -iter [-reject] + +# changes the state of the detrun. -rerun cause a new iteration to be started +# with the accepted with just the exp_ids +dettool -updatedetrun -det_id -iter -stat F32 -stat_stdev [-rerun|-ok|-stop] + +# manual starts a new detrun iteration starting from the specified -iter +dettool -rerun -det_id -iter -accept expid -accept expid -reject expid -reject expid +dettool -rerun -det_id -iter + +# detInputExp needs to track... +det_id iter exp_id use bool accept bool + + + +dettool + + -define | creates a a new detrend Run + input: + exp IDs? + type of det run? + creates a new detRun ID automatically + + output: the new det_id + + -raw | lists raw imfiles needing to be processed + input: + det ID + exp ID + class ID + iteration number? + output: list of unprocessed imfiles + + -addprocessed? | marks a raw imfile as processed + input: + det ID + exp ID + class ID + iteration number? + uri + stat + recipe + output: outout on error only + -processed | lists processed imfiles + input: + det ID + exp ID + class ID + iteration number? + output: list of processed imfiles + -addstacked | adds a stacked imfiles + input: + det ID + class ID + iteration number? + URI + recipe + stats + output: output on error only + -stacked | lists stacked imfiles + input: + det ID + class ID + iteration number? + output: list of stacked imfiles + -stacked | lists stacked imfiles + input: + det ID + class ID + iteration number? + output: list of stacked imfiles + -stackedframe | lists stacked imfiles for COMPLETE frames + input: + det ID // required for simplicity + iteration number? + output: list of stacked imfiles + -addmaster | adds a master imfiles + input: + det ID + class ID + iteration number? + URI + recipe + stats + output: output on error only + -master | lists master imfiles + input: + det ID // required for simplicity + class ID + iteration number? + output: a list of master imfiles + -masterframe | lists master imfiles for COMPLETE frames + input: + det ID // required for simplicity + iteration number? + output: a list of master imfiles Index: /branches/eam_branches/eam_branch_20080918/ippTools/doc/dsfilesetls =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/doc/dsfilesetls (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/doc/dsfilesetls (revision 22161) @@ -0,0 +1,6 @@ +#!/usr/bin/env perl + +print "# uri fileid bytes md5sum type \n"; +print "http://example.org/1 foo1.0 123 asadfasdfasdfasfdasdf CELL\n"; +print "http://example.org/2 foo1.1 123 asadfasdfasdfasdfasdf CELL\n"; +print "http://example.org/3 foo1.2 123 asadfasdfasdfasdfasdf CELL\n"; Index: /branches/eam_branches/eam_branch_20080918/ippTools/doc/dsproductls =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/doc/dsproductls (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/doc/dsproductls (revision 22161) @@ -0,0 +1,8 @@ +#!/usr/bin/env perl + +print "# uri fileset datetime type\n"; +print "http://example.org/1 foo1 2006-04-17T00:00:12 OBJECT\n"; +print "http://example.org/2 foo2 2006-04-17T00:00:12 OBJECT\n"; +print "http://example.org/3 foo3 2006-04-17T00:00:12 OBJECT\n"; + + Index: /branches/eam_branches/eam_branch_20080918/ippTools/doc/install.pod =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/doc/install.pod (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/doc/install.pod (revision 22161) @@ -0,0 +1,251 @@ +=pod + +=head1 SYNOPSIS + +C is a collection of utilties for tracking image processing +tasks. Very little decision making logic is built into these programs. The +intent is for higher level programs and scripts to make the real I +and then use C to encode those choices in a persistent manner. + +=head1 INSTALL + +=head2 Prerequisites + +C depends on C (built with C support), C, and +C and requires a C capable compiler. + +XXX document required library versions + +=head2 Build Procedure + +C is built with the so-called suite of autotools. The build +procedure is pretty typical for software built with these tools. E.g. + + ./configure + make + make install + +If you checked out the sources directly from CVS you need to run the +C script. C will run C for you so that +changes the build procedure to just: + + sh autogen.sh + make + make install + +=head2 jhbuild Build Procedure + +If you are using one of the C modulesets jhbuild can build and install C for you. E.g. + + jhbuild build pxtools + +=head2 Database Setup + +All of the C' persistent storage is ultimately done through C's +C facilties. In turn, C uses C to impliment storage (and +relational querying). C requires that each namespace (aka database) is +configured with basic access controls. These include usernames, passwords, +access filtering, and privilege restrictions. Below is a simple (but somewhat +dangerous) example of configuring a database named C. + + mysql -u root mysql -p + (enter your MySQL root password) + + DROP DATABASE IF EXISTS ipp; + CREATE DATABASE ipp; + GRANT ALL PRIVILEGES ON *.* TO 'ippuser'@'localhost' IDENTIFIED BY 'ipppass'; + FLUSH PRIVILEGES; + quit + +After setting up the database you should verify that you can connect to with the username and password you configured + + mysql -u ipp ipp -pipp + +Next, C needs to be told where to to find MySQL, the name of the +database you created, the username, and the password you created. C +uses the typical C configuration system. The simplest method is to +add the follow lines to your $HOME/.ipprc file. + + ### Database configuration + DBSERVER STR localhost # Database host name (for psDBInit) + DBNAME STR ipp + DBUSER STR ipp # Database user name (for psDBInit) + DBPASSWORD STR ipp # Database password (for psDBInit) + +The C utilities require the a schema is pre-loaded into the database. The C tool can do this for you. E.g. + + pxadmin -create + +You should always verify that the scheme was actually loaded. E.g. + + mysql -u ipp ipp -pipp + Welcome to the MySQL monitor. Commands end with ; or \g. + Your MySQL connection id is 6 to server version: 4.1.20-log + + Type 'help;' or '\h' for help. Type '\c' to clear the buffer. + + mysql> show tables; + +------------------------+ + | Tables_in_ipp | + +------------------------+ + | detInputExp | + | detMasterFrame | + | detMasterImfile | + | detNormalizedImfile | + | detProcessedImfile | + | detResidExpAnalysis | + | detResidImfileAnalysis | + | detRun | + | detStackedImfile | + | newExp | + | newImfile | + | p1PendingExp | + | p2DoneExp | + | p2DoneImfile | + | p2PendingExp | + | p2PendingImfile | + | p3PendingExp | + | pzPendingExp | + | pzPendingImfile | + | rawDetrendExp | + | rawImfile | + | rawScienceExp | + | summitExp | + +------------------------+ + 23 rows in set (0.00 sec) + +=head1 USE + +The workflow is as follows: + +XXX + +=head2 Phase 0 tools + +The primary C utility for handling Phase 0 is C. This tool +has two modes C<-pending> and C<-update>. The pending mode lists all +Cs that are part of an exposure that I yet been classified +as either science or detrend data. + +For example: + + p0search -pending + +Would output something like: + + newImfile MULTI # + + newImfile METADATA + exp_id STR 10 + class STR OTA + class_id STR 0 + uri STR file://0 + END + + newImfile METADATA + exp_id STR 10 + class STR OTA + class_id STR 1 + uri STR file://1 + END + + newImfile METADATA + exp_id STR 10 + class STR OTA + class_id STR 2 + uri STR file://2 + END + + newImfile METADATA + exp_id STR 10 + class STR OTA + class_id STR 3 + uri STR file://3 + END + +To declare C as science data, you would invoke C like this: + + p0search -update -exp_id 10 + +Note that by default C<-update> flags the exposure as science data. To declare it to be detrend data the command would be: + + p0search -update -exp_id 10 -detrend + +Also keep in mind that this is a one time declaration. Once it has been +declared as either science or detrend data this decision can not be undone +(without manually editing the database) + +=head2 Phase 1 tools + +These tools are not yet in use. + +=head2 Phase 2 tools + +C handles all of state for Phase 2. It has four basic modes: + +=over 4 + +=item * C<-quick> + +=item * C<-define> + +=item * C<-pending> + + p0search -pending + +=item * C<-done> + +=head2 Administrative tools + +=over 4 + +=item * C + +This tool can be used to load/unload the database schema. It has three modes of operation: + +=over 4 + +=item * C<-create> + +Loads the database schema. + +=item * C<-delete> + +Deletes the database schema. + +=item * C<-recreate> + +Deletes the database schema then reloads it. + +=back + +=item * C + +I data directly into the database. It has one mode parameter for each database table. Note: support for only a few tables has been impliemented. + +=back + +=head2 Bypassing 'Summit Copy' + +The C suite assumes that exposures and image data are initially +registered with the C set of I tools. Sometimes this is +inconvenient as you want to work with local data and bypass this set. This is +what the C utility is for. C directly inserts data into +the database, side stepping any ordering rules or sanity checking. + +The end result of the C utilities is that new exposures are registered in the C table and raw image data is in the C table. You can insert this data yourself, preparing the system to run I. + +Here is an example of inserting an exposure that has 4 component image files. + + #!//bin/sh + + inject="./pxinject" + + $inject -newExp -exp_id 10 -inst gpc -telescope ps1 -exp_type object -imfiles 4 + + `$inject -newImfile -exp_id 10 -class OTA -class_id 0 -uri file://0` + `$inject -newImfile -exp_id 10 -class OTA -class_id 1 -uri file://1` + `$inject -newImfile -exp_id 10 -class OTA -class_id 2 -uri file://2` + `$inject -newImfile -exp_id 10 -class OTA -class_id 3 -uri file://3` + +=cut Index: /branches/eam_branches/eam_branch_20080918/ippTools/doc/magic_flow.txt =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/doc/magic_flow.txt (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/doc/magic_flow.txt (revision 22161) @@ -0,0 +1,65 @@ +### Set up a run: +magictool -definerun -workdir /path/to/workdir +magictool -addinputskyfile -magic_id 6 -diff_id 1 -node 5.1 +magictool -addinputskyfile -magic_id 6 -diff_id 2 -node 5.2 +magictool -addinputskyfile -magic_id 6 -diff_id 3 -node 5.3 +magictool -addinputskyfile -magic_id 6 -diff_id 4 -node 5.4 +magictool -updaterun -magic_id 6 -state run + +### Set up a run in a single call: +magictool -queue -select_XXX + +### Generate a processing tree +magictool -totree +#magic_tree.pl --magic_id X [camera, tess_id, ra0, dec0] +* magictool -inputskyfile -magic_id 6 +* magictool -inputtree -magic_id 6 -dep_file magic_dep.mdc + +### Process the tree +magictool -toprocess +#magicMe METADATA +# magic_id S64 5 +# node STR 5.1 +# dep STR NULL +# done S64 0 +# uri STR NULL +#END +#magic_process.pl --magic_id 6 --node 5.1 +magictool -inputs -magic_id 6 -node 5.1 + +magictool -addresult -magic_id 6 -node 5.1 -uri foo +magictool -addresult -magic_id 6 -node 5.2 -uri bar +magictool -addresult -magic_id 6 -node 5.3 -uri baz +magictool -addresult -magic_id 6 -node 5.4 -uri qix + +### Merging the tree +magictool -toprocess +#magicMe METADATA +# magic_id S64 5 +# node STR 5A +# uri MULTI +# uri STR foo +# uri STR bar +#END +#magic_process.pl --magic_id 6 --node 5A +magictool -inputs -magic_id 6 -node 5A + +magictool -addresult -magic_id 6 -node 5A -uri foobar +magictool -addresult -magic_id 6 -node 5B -uri bazqix + +### Final merge +magictool -toprocess +#magicMe METADATA +# magic_id S64 5 +# node STR root +#END +#magic_process.pl --magic_id 6 --node root +magictool -inputs -magic_id 6 -node root + +magictool -addresult -magic_id 6 -node root -uri foobarbazqix + +### Generate the mask description +magictool -tomask +magic_mask.pl --magic_id X +magictool -inputs -magic_id 6 -node root +magictool -addmask -magic_id 6 -uri foobarbazqixqit Index: /branches/eam_branches/eam_branch_20080918/ippTools/doc/p0tools.txt =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/doc/p0tools.txt (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/doc/p0tools.txt (revision 22161) @@ -0,0 +1,40 @@ + +Phase 0 pipeline tools: + +p0search -pending : + * examine the new.imfiles,new.exposures tables and select exposures ready for analysis + * output is: (expID) (camera) + +p0search -update (expID) (camera): + * select a the corresponding images from the new.imfiles/new.exposures table + * extract the specified header information + * search the summit metadata db tables + * write an entry to the raw.imfiles and raw.exposure tables + * set the state on the new.imfiles,new.exposure tables + * based on the camera config information; + * add an entry to the p1.pending table (mosaic) + - or + * add an entry to the p2.pending table (single) + +p0search -stats (expID) (camera): + * select a specified image in the new.imfiles/new.exposures table + * extract the specified header information + * search the summit metadata db tables + * report the image stats + [-update without output to MDDB] + +p0search -mkraw (expID) (camera): + * select a specified image in the new.imfiles/new.exposures table + * extract the specified header information + * search the summit metadata db tables + * write an entry to the raw.imfiles and raw.exposure tables + * set the state on the new.imfiles,new.exposure tables + +p0search -cleanup: + * remove completed entries from the new.imfiles,new.exposure tables + +** note : the division of -pending and -update allows separate processes + to be examining the image headers and measuring some stats. these + jobs can be run via pcontrol to reduce the load on the PanTasks + machines + Index: /branches/eam_branches/eam_branch_20080918/ippTools/doc/p1tools.txt =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/doc/p1tools.txt (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/doc/p1tools.txt (revision 22161) @@ -0,0 +1,16 @@ + +Phase 1 pipeline tools: + +p1search -define [constraints]: + * examine the raw.exposures tables and select exposures matching the given criteria + * add entries which are allowed (mosaic) to the p1.pending table + +p1search -pending : + * examine the p1.pending table and select exposures waiting for p1 + * output: lines consisting of: + (expID) (p1version) (camera) + +p1search -done : + * select completed entries in the p1.pending table + * move to the p1.done table + * add new entry to the p2.pending tables Index: /branches/eam_branches/eam_branch_20080918/ippTools/doc/p2search.sh =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/doc/p2search.sh (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/doc/p2search.sh (revision 22161) @@ -0,0 +1,8 @@ +#!/bin/csh -f + +echo "# expID class classID url" +echo "654320o fpa megacam 654320o.fits" +echo "654321o fpa megacam 654321o.fits" +echo "654322o fpa megacam 654322o.fits" +echo "654323o fpa megacam 654323o.fits" +echo "654324o fpa megacam 654324o.fits" Index: /branches/eam_branches/eam_branch_20080918/ippTools/doc/p2tools.txt =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/doc/p2tools.txt (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/doc/p2tools.txt (revision 22161) @@ -0,0 +1,41 @@ + +Phase 2 pipeline tools: + +p2search -quick + * search for images which match in raw.exp,raw.imfiles + * output in format which can be used by ppImage pantasks script + +p2search -define [options] + * input: searches mddb:raw_exposures,raw_images + - compares against p2Pending[Exp|Imfile] + - compares against p2Done[Exp|Imfile] + - XXX should be implemented as a special wrapper function that + implements this comparison as a SQL query + * output: updates mddb:P2_exposures_pending,P2_images_pending + * alternative output: identical to p2pending + +p2search -pending + * input: searches mddb:P2_exposures_pending,P2_images_pending + * output: Nlines consisting of: + (URL) (expID) (class) + * options: ? + +p2search -update + * examine the imfiles and identify any completed exposures + +p2search -done + * get exp_id/class/class_id from the CLI + * add the completed imfile to the p2DoneImfile tables + * remove corresponding entries from the p2PendingImfile table + * check to see if any p2PendingExps have no associated p2PendingImfiles + if so move the p2PendingExp(s) to p2DoneExp +// * send new entry to the pending p3 table + +ppImage file://path/filename file://path/outroot -recipe (recipe) +ppImage neb://nebname neb://outroot -recipe (recipe) + +restriction options: + -time (start) (stop) + -camera (camera) + -region (ra,dec) (ra,dec) + Index: /branches/eam_branches/eam_branch_20080918/ippTools/doc/p3tools.txt =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/doc/p3tools.txt (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/doc/p3tools.txt (revision 22161) @@ -0,0 +1,22 @@ + +Phase 3 pipeline tools: + +p3search -define : + * examine the raw.exposures tables and select exposures matching the given criteria + * add entries which are allowed (mosaic) to the p3.pending table + +p3search -quick : + * examine the raw.exposures tables and select exposures matching the given criteria + * return list of entries for p3 processing + * output: lines consisting of: + (expID) (p3version) (camera) + +p3search -pending : + * examine the p3.pending exposures table and select exposures waiting for p3 + * return list of entries for p3 processing + * output: lines consisting of: + (expID) (p3version) (camera) + +p3search -done : + * select completed entries in the p3.pending table + * move to the p3.done table Index: /branches/eam_branches/eam_branch_20080918/ippTools/doc/summit_copy.txt =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/doc/summit_copy.txt (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/doc/summit_copy.txt (revision 22161) @@ -0,0 +1,49 @@ +The summit copy phase (phase Z or pz) consists of three basic steps. +Discovering what exposures and available, discovering what image files are +associated with those exposures, and the downloading of image files. All +"external" communication with the DataStore is done with three utilities; +pzgetexp, pzgetimfiles & dsget. While local state information is accessed and +updated via the 'pztool' utility. + +The basic work follow is: + +pzgetexp retrieves a list of all of the exposures (filesets) available from a +DataStore "product". If any previously unknown file exposures are found they +are added to the pzPendingExp tables and the summitExp table. New entries to +the summitExp table have a imfile value of NULL as this information can't yet +be known. If any exposures are already in the summitExp table the most recent +exp_id is used to constrain the result set requested from the DataStore. + + ./pzgetexp -uri http://otis/ds/skyprobe/ -inst skyprobe -telescope ps1 + +The results of pzgetexp and accessible with pztool -pendingexp`. Which will +return a list of new exposures from the pzPendingExp table. + + ./pztool -pendingexp + +The output of which is used to invoke pzgetimfiles. + +pzgetimfiles retrieves a list of all of the imfiles (files) in an exposure +(fileset) from the DataStore. It then updates the imfiles count in the +summitExp table and adds each imfile to the pzPendingImfile table. + +./pzgetimfiles -uri http://otis/ds/skyprobe/sep8_twi_05/ -filesetid sep8_twi_05 -inst skyprobe -telescope ps1 + +pztool -pendingimfile returns the list of imfiles that need to be downloaded. + + ./pztool -pendingimfile + +The results of which are used to invoke dsget and pztool -copydone after dsget +completes it's download. + +dsget from the 'DataStore' package takes the follow parameters. + + dsget --uri --filename [--bytes ] [--md5 ] + +Where "filename" may be either just a filename relative to the CWD or a fully +qualified path. After dsget has copied the file pztool -copydone needs to be +invoked + + ./pztool -copydone -exp_id sep8_twi_03 -inst skyprobe -telescope ps1 -class chip -class_id sep8_twi_03.fits -uri file///data/alala/sep8_twi_03.fits + +And then we're Pau. Index: /branches/eam_branches/eam_branch_20080918/ippTools/m4/ac_prog_perl_modules.m4 =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/m4/ac_prog_perl_modules.m4 (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/m4/ac_prog_perl_modules.m4 (revision 22161) @@ -0,0 +1,53 @@ +dnl @synopsis AC_PROG_PERL_MODULES([MODULES], [ACTION-IF-TRUE], [ACTION-IF-FALSE]) +dnl +dnl Checks to see if the the given perl modules are available. If true +dnl the shell commands in ACTION-IF-TRUE are executed. If not the shell +dnl commands in ACTION-IF-FALSE are run. Note if $PERL is not set (for +dnl example by calling AC_CHECK_PROG, or AC_PATH_PROG), +dnl AC_CHECK_PROG(PERL, perl, perl) will be run. +dnl +dnl Example: +dnl +dnl AC_CHECK_PERL_MODULES(Text::Wrap Net::LDAP, , +dnl AC_MSG_WARN(Need some Perl modules) +dnl +dnl @category InstalledPackages +dnl @author Dean Povey +dnl @version 2002-09-25 +dnl @license AllPermissive + +AC_DEFUN([AC_PROG_PERL_MODULES],[ + ac_perl_modules="$1" + # Make sure we have perl + if test -z "$PERL"; then + AC_CHECK_PROG(PERL,perl,perl) + fi + + if test "x$PERL" != x; then + ac_perl_modules_failed=0 + for ac_perl_module in $ac_perl_modules; do + AC_MSG_CHECKING(for perl module $ac_perl_module) + + # Would be nice to log result here, but can't rely on autoconf + # internals + $PERL "-M$ac_perl_module" -e exit > /dev/null 2>&1 + if test $? -ne 0; then + AC_MSG_RESULT(no); + ac_perl_modules_failed=1 + else + AC_MSG_RESULT(ok); + fi + done + + # Run optional shell commands + if test "$ac_perl_modules_failed" = 0; then + : + $2 + else + : + $3 + fi + else + AC_MSG_WARN(could not find perl) + fi +]) Index: /branches/eam_branches/eam_branch_20080918/ippTools/notes.txt =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/notes.txt (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/notes.txt (revision 22161) @@ -0,0 +1,63 @@ + +2008.07.19 EAM + + * need to add cleanup modes to all stages. here is the minimum list + of stages that need the cleanups: + + * chiptool - DONE + * camtool + * faketool + * warptool + * difftool + * stacktool + * dettool processedimfile + * dettool processedexp + * dettool stackedimfile + * dettool normalizedstat + * dettool normalizedimfile + * dettool normalizedexp + * dettool residimfile + * dettool residexp + + * I am modifiying the tools to use the new states: new, full, goto_cleaned, etc. + I am NOT changing: regtool, pztool, magictool, flatcorr, pxregister, pstamptool + + * for dettool, I have added a new field, 'data_state' to the + detRunSummary. My plan here is that a complete detRunIteration + is cleaned at once. I've added functions to all of the dettool + major modes to list pending entries to be cleaned and a function + to update the data_state when cleaning is done. + +2008.07.11 EAM + + * break dettool.c into files for each detrend stage + * require -iteration in more places? (esp -add...) + * allow %s, %d arguments in the *.sql files or not? + * adddetrunsummary has the ability to set values for multiple detRunSummary entries. why? + +2008.07.09 EAM + + In order to implement the 'cleanup' and 'update' strategies, we have + changed the state names and defined addtional states as follows: + + * new : entry is unprocessed (was 'run') + * full : entry is processed and has all output data products (was 'done') + * goto_cleaned : entry should be processed by the 'cleanup' system + * cleaned : entry has been processed by the 'cleanup' system -- + some output data products are now missing. + + * goto_full : entry should be re-processed by the 'update' system + * full : entry has been re-processed by the 'update' system + + * goto_purged : entry should be processed by the 'cleanup' ssytem + to purge all output data + * purged : entry has been purged + + + +2008.05.16 EAM + +caltool + * add the active field, -active flag to -dbs + * add the -region + * change 'catdir' to 'dvo_id' Index: /branches/eam_branches/eam_branch_20080918/ippTools/pxtools.pc.in =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/pxtools.pc.in (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/pxtools.pc.in (revision 22161) @@ -0,0 +1,11 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@/@PACKAGE_NAME@ + +Name: @PACKAGE_NAME@ +Description: Pan-STARRS IPP Tools Library +Version: @VERSION@ +Requires: pslib psmodules ippdb +Libs: -L${libdir} -lpxtools +Cflags: -I${includedir} @PXTOOLS_CFLAGS@ Index: /branches/eam_branches/eam_branch_20080918/ippTools/scripts/.cvsignore =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/scripts/.cvsignore (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/scripts/.cvsignore (revision 22161) @@ -0,0 +1,2 @@ +Makefile +Makefile.in Index: /branches/eam_branches/eam_branch_20080918/ippTools/scripts/camtest.sh =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/scripts/camtest.sh (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/scripts/camtest.sh (revision 22161) @@ -0,0 +1,11 @@ +#!//bin/sh + +set -o verbose + +chiptest.sh || exit 1 + +camtool -pendingexp || exit 1 +camtool -pendingimfile || exit 1 + +camtool -addprocessedexp -cam_id 1 -uri file:///cam -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -sigma_ra 1 -sigma_dec 2 -zp_mean 10 -zp_stdev 2 -n_stars 2 -n_extended 0 -n_astrom 42 -n_cr 10000000 -path_base file:///foo || exit 1 +camtool -addprocessedexp -cam_id 2 -uri file:///cam -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -sigma_ra 1 -sigma_dec 2 -zp_mean 10 -zp_stdev 2 -n_stars 2 -n_extended 0 -n_astrom 42 -n_cr 10000000 -path_base file:///foo || exit 1 Index: /branches/eam_branches/eam_branch_20080918/ippTools/scripts/chiptest.sh =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/scripts/chiptest.sh (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/scripts/chiptest.sh (revision 22161) @@ -0,0 +1,17 @@ +#!//bin/sh + +set -o verbose + +regtest.sh || exit 1 + +chiptool -pendingimfile || exit 1 + +for ID in `seq 0 3`; do + chiptool -addprocessedimfile -chip_id 1 -exp_id 1 -class_id $ID -uri file://chipp-t10.$ID -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file:///foo || exit 1 +done; + +for ID in `seq 0 3`; do + chiptool -addprocessedimfile -chip_id 2 -exp_id 2 -class_id $ID -uri file://chipp-t11.$ID -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file:///foo || exit 1 +done; + +chiptool -pendingimfile || exit 1 Index: /branches/eam_branches/eam_branch_20080918/ippTools/scripts/dettest.sh =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/scripts/dettest.sh (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/scripts/dettest.sh (revision 22161) @@ -0,0 +1,88 @@ +#!//bin/sh + +set -o verbose + +det_id=1 + +./regtest.sh -end_stage reg || exit 1 + +det_id=`dettool -definebyquery -det_type bias -inst gpc -filelevel fpa -select_exp_type bias -airmass_min 1 -airmass_max 10 -exp_time_min 10 -exp_time_max 30.0 -workdir file::///some/path -simple | cut -f1 -d" "` || exit 1 + +dettool -raw || exit 1 + +dettool -toprocessedimfile || exit 1 + +for ID in `seq 0 3` ; do + dettool -addprocessedimfile -det_id 1 -exp_id 1 -class_id $ID -uri file://proc-$ID -recip myrecip -bg 2 -bg_stdev 3 -bg_mean_stdev 4 || exit 1 +done; + +for ID in `seq 0 3` ; do + dettool -addprocessedimfile -det_id 1 -exp_id 2 -class_id $ID -uri file://proc-$ID -recip myrecip -bg 2 -bg_stdev 3 -bg_mean_stdev 4 || exit 1 +done; + + +dettool -tostacked || exit 1 + +for ID in `seq 0 3` ; do + dettool -addstacked -det_id $det_id -uri file://stacked-$ID -class_id $ID -recip myrecipe -bg 1 -bg_stdev 2 -bg_mean_stdev 3 || exit 1 +done; + +dettool -tonormalizedstat || exit 1 + +for ID in `seq 0 3` ; do + dettool -addnormalizedstat -det_id $det_id -class_id $ID -norm 0.12345 || exit 1 +done; + +dettool -tonormalize || exit 1 +for ID in `seq 0 3` ; do + dettool -addnormalizedimfile -det_id $det_id -class_id $ID -uri file://normalized-$ID -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base banana1 || exit 1 +done; + +dettool -tonormalizedexp || exit 1 +dettool -addnormalizedexp -det_id $det_id -recip myrecipe -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file://normalizedexp || exit 1 + +dettool -toresidimfile || exit 1 + +for ID in `seq 0 3` ; do + dettool -addresidimfile -det_id $det_id -exp_id 1 -class_id $ID -recip myrecip -bg 1 -bg_stdev 1 -bg_mean_stdev 1 -uri file://resid-$ID || exit 1 +done; + +for ID in `seq 0 3` ; do + dettool -addresidimfile -det_id $det_id -exp_id 2 -class_id $ID -recip myrecip -bg 1 -bg_stdev 1 -bg_mean_stdev 1 -uri file://resid-$ID || exit 1 +done; + +dettool -toresidexp || exit 1 +dettool -addresidexp -det_id $det_id -exp_id 1 -recip myrecipe -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base jpeg1 || exit 1 +dettool -addresidexp -det_id $det_id -exp_id 2 -recip myrecipe -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base jpeg1 -reject || exit 1 + +dettool -todetrunsummary || exit 1 +dettool -residexp || exit 1 +dettool -updateresidexp -det_id $det_id -iteration 0 -recip yourrecipe || exit 1 +dettool -updateresidexp -det_id $det_id -iteration 0 -exp_id 2 -reject || exit 1 +dettool -updateresidexp -det_id $det_id -iteration 0 -exp_id 2 || exit 1 + +dettool -adddetrunsummary -det_id $det_id -iteration 0 -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -accept || exit 1 + +dettool -todetrunsummary || exit 1 + +dettool -residexp || exit 1 + +dettool -updatedetrun -det_id $det_id -state stop || exit 1 + +dettool -register_detrend -det_type bias -mode master -filelevel fpa -workdir file:/// -label foo || exit 1 +dettool -register_detrend_imfile -det_id 2 -class_id 1 -uri file:///foo || exit 1 +dettool -register_detrend_imfile -det_id 2 -class_id 2 -uri file:///foo || exit 1 +dettool -register_detrend_imfile -det_id 2 -class_id 3 -uri file:///foo || exit 1 +dettool -register_detrend_imfile -det_id 2 -class_id 4 -uri file:///foo || exit 1 +dettool -updatedetrun -det_id 2 -state stop || exit 1 + +# correct test +dettool -makecorrection -det_id 1 || exit 1 +dettool -tocorrectexp || exit 1 +dettool -tocorrectimfile -det_id 3 || exit 1 +dettool -addcorrectimfile -det_id 3 -class_id 0 -uri file:///correct/0 || exit 1 +dettool -addcorrectimfile -det_id 3 -class_id 1 -uri file:///correct/1 || exit 1 +dettool -addcorrectimfile -det_id 3 -class_id 2 -uri file:///correct/2 || exit 1 +dettool -addcorrectimfile -det_id 3 -class_id 3 -uri file:///correct/3 || exit 1 +# detRun 3 should be automatically set to stop by this point +dettool -tocorrectexp || exit 1 Index: /branches/eam_branches/eam_branch_20080918/ippTools/scripts/difftest.sh =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/scripts/difftest.sh (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/scripts/difftest.sh (revision 22161) @@ -0,0 +1,29 @@ +#!//bin/sh + +set -o verbose + +warptest.sh || exit 1 + + +difftool -definerun -workdir file:///tmp/diff -skycell_id foo1 -tess_id bar || exit 1 +difftool -addinputskyfile -diff_id 1 -warp_id 1 -kind warped -template || exit 1 +difftool -addinputskyfile -diff_id 1 -warp_id 1 -kind warped || exit 1 +#difftool -updaterun -state run -diff_id 1 || exit 1 +difftool -todiffskyfile || exit 1 +difftool -inputskyfile || exit 1 +difftool -adddiffskyfile -diff_id 1 -uri file:///tmp/diff/skyfile -path_base file://lalaland -bg 1 -bg_stdev 2 || exit 1 +difftool -diffskyfile -diff_id 1 || exit 1 +difftool -updaterun -state stop -diff_id 1 || exit 1 + +# +# diff_id 2 +# +difftool -definerun -workdir file:///tmp/diff -skycell_id foo1 -tess_id bar || exit 1 +difftool -addinputskyfile -diff_id 2 -warp_id 1 -kind warped -template || exit 1 +difftool -addinputskyfile -diff_id 2 -warp_id 1 -kind warped || exit 1 +#difftool -updaterun -state run -diff_id 1 || exit 1 +difftool -todiffskyfile || exit 1 +difftool -inputskyfile || exit 1 +difftool -adddiffskyfile -diff_id 2 -uri file:///tmp/diff/skyfile -path_base file://lalaland -bg 1 -bg_stdev 2 || exit 1 +difftool -diffskyfile -diff_id 2 || exit 1 +difftool -updaterun -state stop -diff_id 2 || exit 1 Index: /branches/eam_branches/eam_branch_20080918/ippTools/scripts/faketest.sh =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/scripts/faketest.sh (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/scripts/faketest.sh (revision 22161) @@ -0,0 +1,19 @@ +#!//bin/sh + +set -o verbose + +camtest.sh || exit 1 + +#faketool -pendingexp || exit 1 +faketool -pendingimfile || exit 1 + +for ID in `seq 0 3`; do + faketool -addprocessedimfile -fake_id 1 -exp_id 1 -class_id $ID -uri file://chipp-t10.$ID -path_base file:///foo || exit 1 +done; + +for ID in `seq 0 3`; do + faketool -addprocessedimfile -fake_id 2 -exp_id 2 -class_id $ID -uri file://chipp-t11.$ID -path_base file:///foo || exit 1 +done; + +faketool -pendingimfile || exit 1 +faketool -processedimfile || exit 1 Index: /branches/eam_branches/eam_branch_20080918/ippTools/scripts/magic_dep.md =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/scripts/magic_dep.md (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/scripts/magic_dep.md (revision 22161) @@ -0,0 +1,6 @@ +# node dep +a STR NULL +b STR NULL +root MULTI +root STR a +root STR b Index: /branches/eam_branches/eam_branch_20080918/ippTools/scripts/magictest.sh =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/scripts/magictest.sh (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/scripts/magictest.sh (revision 22161) @@ -0,0 +1,28 @@ +#!//bin/sh + +set -o verbose + +difftest.sh || exit 1 + + +magictool -definerun -workdir file:///foo/bar/ || exit 1 +magictool -addinputskyfile -magic_id 1 -diff_id 1 -node a || exit 1 +magictool -addinputskyfile -magic_id 1 -diff_id 2 -node b || exit 1 +magictool -inputtree -magic_id 1 -dep_file magic_dep.md +magictool -updaterun -state run -magic_id 1 || exit 1 + +magictool -addresult -magic_id 1 -node a -uri file:///foo/a +magictool -addresult -magic_id 1 -node b -uri file:///foo/b +magictool -addresult -magic_id 1 -node root -uri file:///foo/root +magictool -tomask +magictool -addmask -magic_id 1 -uri file:///foo/mask +magictool -toskyfilemask +magictool -addskyfilemask -magic_id 1 -diff_id 1 -uri file:///foo/mask_1 +magictool -addskyfilemask -magic_id 1 -diff_id 2 -uri file:///foo/mask_2 +magictool -updaterun -state stop -magic_id 1 || exit 1 + +#difftool -todiffskyfile || exit 1 +#difftool -inputskyfile || exit 1 +#difftool -adddiffskyfile -diff_id 1 -uri file:///tmp/diff/skyfile -path_base file://lalaland -bg 1 -bg_stdev 2 || exit 1 +#difftool -diffskyfile -diff_id 1 || exit 1 +#difftool -updaterun -state stop -diff_id 1 || exit 1 Index: /branches/eam_branches/eam_branch_20080918/ippTools/scripts/mapfile.txt =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/scripts/mapfile.txt (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/scripts/mapfile.txt (revision 22161) @@ -0,0 +1,19 @@ +warpSkyCellMap MULTI + +warpSkyCellMap METADATA + warp_id S64 1 + cam_id S64 1 + skycell_id STR foo1 + tess_id STR bar + class_id STR quix1 + fault S16 0 +END + +warpSkyCellMap METADATA + warp_id S64 1 + cam_id S64 1 + skycell_id STR foo2 + tess_id STR bar + class_id STR quix2 + fault S16 0 +END Index: /branches/eam_branches/eam_branch_20080918/ippTools/scripts/regtest.sh =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/scripts/regtest.sh (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/scripts/regtest.sh (revision 22161) @@ -0,0 +1,49 @@ +#!//bin/sh + +set -o verbose + +inject="pxinject" +regtool="regtool" + +echo -e "YES\njhipp\n\n" | pxadmin -recreate || exit 1 + +exp_id1=`$inject -newExp -tmp_exp_name t10 -tmp_inst gpc -tmp_telescope ps1 -workdir file::///some/path -reduction batman -simple` || exit 1 + +echo $exp_id1 + +for ID in `seq 0 3`; do + $inject -newImfile -exp_id $exp_id1 -tmp_class_id tmp.$ID -uri file://$ID || exit 1 +done; + +$inject -updatenewExp -exp_id $exp_id1 -state run + +exp_id2=`$inject -newExp -tmp_exp_name t11 -tmp_inst gpc -tmp_telescope ps1 -workdir file::///some/path -reduction robin -simple` || exit 1 + +for ID in `seq 0 3`; do + $inject -newImfile -exp_id $exp_id2 -tmp_class_id tmp.$ID -uri file://$ID || exit 1 +done; + +$inject -updatenewExp -exp_id $exp_id2 -state run + + +$regtool -pendingimfile || exit 1 + +for ID in `seq 0 3`; do + $regtool -addprocessedimfile -exp_id $exp_id1 -exp_name t10 -inst gpc -telescope ps1 -tmp_class_id tmp.$ID -class_id $ID -filter r -airmass 10 -ra 1 -decl 2 -exp_time 0 -bg 1 -bg_stdev 1 -bg_mean_stdev 10 -alt 10 -az 10 -ccd_temp 10 -posang 10 -object dog -dateobs 2006-10-20T10:10:10Z -uri file://$ID || exit 1 +done; + +$regtool -pendingexp|| exit 1 + +$regtool -addprocessedexp -exp_id $exp_id1 -exp_name t10 -inst gpc -telescope ps1 -exp_tag batman.t10 -filelevel OTA -filter r -airmass 10 -ra 1 -decl 2 -exp_type object -exp_time 0 -bg 10 -bg_stdev 1 -bg_mean_stdev 10 -alt 10 -az 10 -ccd_temp 45 -posang 10 -object dog -dateobs "2006-10-20T10:10:10Z" -label foobar -tess_id moldyshoe $* || exit 1 +# use BIAS or OBJECT? $regtool -addprocessedexp -exp_id $exp_id1 -exp_name t10 -inst gpc -telescope ps1 -exp_tag batman.t10 -filelevel OTA -filter r -airmass 10 -ra 1 -decl 2 -exp_type bias -exp_time 0 -bg 10 -bg_stdev 1 -bg_mean_stdev 10 -alt 10 -az 10 -ccd_temp 45 -posang 10 -object dog -dateobs "2006-10-20T10:10:10Z" -label foobar -tess_id moldyshoe $* || exit 1 + +$regtool -pendingimfile || exit 1 + +for ID in `seq 0 3`; do + $regtool -addprocessedimfile -exp_id $exp_id2 -exp_name t11 -inst gpc -telescope ps1 -tmp_class_id tmp.$ID -class_id $ID -filter r -airmass 10 -ra 1 -decl 2 -exp_time 0 -bg 1 -bg_stdev 1 -bg_mean_stdev 10 -alt 10 -az 10 -ccd_temp 10 -posang 10 -object dog -dateobs 2006-10-20T10:10:10Z -uri file://$ID || exit 1 +done; + +$regtool -pendingexp|| exit 1 + +$regtool -addprocessedexp -exp_id $exp_id2 -exp_name t11 -inst gpc -telescope ps1 -exp_tag batman.t11 -filelevel OTA -filter r -airmass 11 -ra 1 -decl 2 -exp_type object -exp_time 0 -bg 11 -bg_stdev 1 -bg_mean_stdev 11 -alt 11 -az 11 -ccd_temp 45 -posang 11 -object dog -dateobs 2006-10-20T10:10:10Z -label foobar -tess_id moldyshoe $* || exit 1 +# BIAS or OBJECT? $regtool -addprocessedexp -exp_id $exp_id2 -exp_name t11 -inst gpc -telescope ps1 -exp_tag batman.t11 -filelevel OTA -filter r -airmass 11 -ra 1 -decl 2 -exp_type bias -exp_time 0 -bg 11 -bg_stdev 1 -bg_mean_stdev 11 -alt 11 -az 11 -ccd_temp 45 -posang 11 -object dog -dateobs 2006-10-20T10:10:10Z -label foobar -tess_id moldyshoe $* || exit 1 Index: /branches/eam_branches/eam_branch_20080918/ippTools/scripts/warptest.sh =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/scripts/warptest.sh (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/scripts/warptest.sh (revision 22161) @@ -0,0 +1,30 @@ +#!//bin/sh + +set -o verbose + +faketest.sh || exit 1 + +#warp tool -definerun -fake_id 1 -mode warp -workdir file:///foo + +#warptool -addinputexp -warp_id 1 -cam_id 1 || exit 1 +#warptool -addinputexp -warp_id 1 -cam_id 2 || exit 1 +#warptool -updaterun -warp_id 1 -state run || exit 1 + +warptool -exp -warp_id 1 || exit 1 + +warptool -imfile -warp_id 1 || exit 1 + +warptool -tooverlap -warp_id 1 || exit 1 + +warptool -addoverlap -mapfile mapfile.txt || exit 1 + +warptool -scmap || exit 1 + +warptool -towarped || exit 1 + +warptool -addwarped -warp_id 1 -skycell_id foo1 -tess_id bar -uri file:///tmp/foo -path_base file://wonderland -bg 1 -bg_stdev 2 || exit 1 +warptool -addwarped -warp_id 1 -skycell_id foo2 -tess_id bar -uri file:///tmp/foo -path_base file://wonderland -bg 1 -bg_stdev 2 || exit 1 + +warptool -warped -warp_id 1 || exit 1 + +#warptool -updaterun -warp_id 1 -state stop || exit 1 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/.cvsignore =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/.cvsignore (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/.cvsignore (revision 22161) @@ -0,0 +1,2 @@ +Makefile.in +Makefile Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/Makefile.am =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/Makefile.am (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/Makefile.am (revision 22161) @@ -0,0 +1,161 @@ +dist_pkgdata_DATA = \ + camtool_donecleanup.sql \ + camtool_find_chip_id.sql \ + camtool_find_pendingexp.sql \ + camtool_find_pendingimfile.sql \ + camtool_find_processedexp.sql \ + camtool_pendingcleanupexp.sql \ + camtool_pendingcleanuprun.sql \ + camtool_queue_chip_id.sql \ + camtool_reset_faulted_runs.sql \ + camtool_revertprocessedexp.sql \ + chiptool_change_exp_state.sql \ + chiptool_change_imfile_data_state.sql \ + chiptool_completely_processed_exp.sql \ + chiptool_donecleanup.sql \ + chiptool_find_rawexp.sql \ + chiptool_pendingcleanupimfile.sql \ + chiptool_pendingcleanuprun.sql \ + chiptool_pendingimfile.sql \ + chiptool_processedimfile.sql \ + chiptool_revertprocessedimfile.sql \ + chiptool_run.sql \ + chiptool_unmasked.sql \ + detselect_search.sql \ + detselect_select.sql \ + dettool_addprocessedexp.sql \ + dettool_childlessrun.sql \ + dettool_definebydetrun.sql \ + dettool_detrunsummary.sql \ + dettool_find_completed_runs.sql \ + dettool_input.sql \ + dettool_normalizedexp.sql \ + dettool_normalizedimfile.sql \ + dettool_normalizedstat.sql \ + dettool_pending.sql \ + dettool_pendingcleanup_normalizedexp.sql \ + dettool_pendingcleanup_normalizedimfile.sql \ + dettool_pendingcleanup_normalizedstat.sql \ + dettool_pendingcleanup_processedexp.sql \ + dettool_pendingcleanup_processedimfile.sql \ + dettool_pendingcleanup_residexp.sql \ + dettool_pendingcleanup_residimfile.sql \ + dettool_pendingcleanup_stacked.sql \ + dettool_processedimfile.sql \ + dettool_raw.sql \ + dettool_residexp.sql \ + dettool_residimfile.sql \ + dettool_revertdetrunsummary.sql \ + dettool_revertnormalizedexp.sql \ + dettool_revertnormalizedimfile.sql \ + dettool_revertnormalizedstat.sql \ + dettool_revertprocessedexp.sql \ + dettool_revertprocessedimfile.sql \ + dettool_revertresidexp.sql \ + dettool_revertresidimfile.sql \ + dettool_revertstacked.sql \ + dettool_runs.sql \ + dettool_stacked.sql \ + dettool_start_new_iteration.sql \ + dettool_stop_completed_correct_runs.sql \ + dettool_tocorrectexp.sql \ + dettool_tocorrectimfile.sql \ + dettool_todetrunsummary.sql \ + dettool_tonormalize.sql \ + dettool_tonormalizedexp.sql \ + dettool_tonormalizedstat.sql \ + dettool_toprocessedexp.sql \ + dettool_toprocessedimfile.sql \ + dettool_toresidexp.sql \ + dettool_toresidimfile.sql \ + dettool_tostacked.sql \ + difftool_definebyquery.sql \ + difftool_donecleanup.sql \ + difftool_inputskyfile.sql \ + difftool_pendingcleanuprun.sql \ + difftool_pendingcleanupskyfile.sql \ + difftool_revertdiffskyfile_delete.sql \ + difftool_revertdiffskyfile_update.sql \ + difftool_skyfile.sql \ + difftool_todiffskyfile.sql \ + faketool_completely_processed_exp.sql \ + faketool_donecleanup.sql \ + faketool_find_camrun.sql \ + faketool_find_pendingexp.sql \ + faketool_pendingcleanupimfile.sql \ + faketool_pendingcleanuprun.sql \ + faketool_pendingimfile.sql \ + faketool_processedimfile.sql \ + faketool_queue_cam_id.sql \ + faketool_revertprocessedimfile.sql \ + faketool_unmasked.sql \ + flatcorr_chiprundone.sql \ + flatcorr_pendingprocess.sql \ + flatcorr_inputimfile.sql \ + magictool_addmask.sql \ + magictool_create_tmp_warpcomplete.sql \ + magictool_definebyquery.sql \ + magictool_definebyquery_insert.sql \ + magictool_definebyquery_select_part1.sql \ + magictool_definebyquery_select_part2.sql \ + magictool_definebyquery_select_test.sql \ + magictool_definebyquery_temp_create.sql \ + magictool_definebyquery_temp_insert.sql \ + magictool_definebyquery_temp_insert_groupby.sql \ + magictool_inputs.sql \ + magictool_inputskyfile.sql \ + magictool_mask.sql \ + magictool_tomask.sql \ + magictool_toprocess_inputs.sql \ + magictool_toprocess_tree.sql \ + magictool_toskyfilemask.sql \ + magictool_totree.sql \ + pstamptool_addjob_otherjob.sql \ + pstamptool_addjob_stampjob.sql \ + pstamptool_datastore.sql \ + pstamptool_pendingjob.sql \ + pstamptool_pendingreq.sql \ + pxadmin_create_tables.sql \ + pxadmin_drop_tables.sql \ + pztool_find_completed_exp.sql \ + pztool_pendingimfile.sql \ + pztool_revert_downloadimfile_faults.sql \ + pztool_revert_fileset_faults.sql \ + pztool_revertcopied.sql \ + regtool_create_dup_table.sql \ + regtool_pendingexp.sql \ + regtool_pendingimfile.sql \ + regtool_populate_dup_table.sql \ + regtool_processedexp.sql \ + regtool_processedimfile.sql \ + regtool_revertprocessedexp.sql \ + regtool_revertprocessedimfile.sql \ + stacktool_definebyquery_insert.sql \ + stacktool_definebyquery_insert_random_part1.sql \ + stacktool_definebyquery_insert_random_part2.sql \ + stacktool_definebyquery_part1.sql \ + stacktool_definebyquery_part2.sql \ + stacktool_definebyquery_test.sql \ + stacktool_donecleanup.sql \ + stacktool_find_complete_warps.sql \ + stacktool_inputskyfile.sql \ + stacktool_pendingcleanuprun.sql \ + stacktool_pendingcleanupskyfile.sql \ + stacktool_revertsumskyfile_delete.sql \ + stacktool_revertsumskyfile_update.sql \ + stacktool_sumskyfile.sql \ + stacktool_tosum.sql \ + warptool_change_skyfile_data_state.sql \ + warptool_change_run_state.sql \ + warptool_definebyquery.sql \ + warptool_donecleanup.sql \ + warptool_exp.sql \ + warptool_imfile.sql \ + warptool_pendingcleanuprun.sql \ + warptool_pendingcleanupskyfile.sql \ + warptool_revertwarped.sql \ + warptool_scmap.sql \ + warptool_tooverlap.sql \ + warptool_towarped.sql \ + warptool_updateskyfile.sql \ + warptool_warped.sql Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_donecleanup.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_donecleanup.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_donecleanup.sql (revision 22161) @@ -0,0 +1,5 @@ +SELECT + camRun.* +FROM camRun +WHERE + camRun.state = 'cleaned' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_find_chip_id.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_find_chip_id.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_find_chip_id.sql (revision 22161) @@ -0,0 +1,31 @@ +SELECT + * +FROM + (SELECT DISTINCT + chipRun.*, + rawExp.camera, + rawExp.telescope, + rawExp.dateobs, + rawExp.exp_tag, + rawExp.exp_type, + rawExp.filelevel, + rawExp.filter, + rawExp.airmass, + rawExp.ra, + rawExp.decl, + rawExp.exp_time, + rawExp.sat_pixel_frac, + rawExp.bg, + rawExp.bg_stdev, + rawExp.bg_mean_stdev, + rawExp.alt, + rawExp.az, + rawExp.ccd_temp, + rawExp.posang, + rawExp.object, + rawExp.solang + FROM chipRun + JOIN rawExp + using(exp_id) + WHERE + chipRun.state = 'full') as Foo Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_find_pendingexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_find_pendingexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_find_pendingexp.sql (revision 22161) @@ -0,0 +1,30 @@ +-- this query is used by both camtool -pendingexp & camtool -addprocessedexp it +-- does a little more work then is necessary for -addprocessed but it seems +-- "cleaner" to use the same query for both cases +SELECT * FROM + (SELECT + camRun.*, + rawExp.exp_tag, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel + FROM camRun + JOIN chipRun + USING(chip_id) +-- JOIN chipProcessedImfile +-- USING(chip_id) + JOIN rawExp + USING(exp_id) +-- ON chipProcessedImfile.exp_id = rawExp.exp_id + LEFT JOIN camProcessedExp + USING(cam_id) + LEFT JOIN camMask + ON camRun.label = camMask.label + WHERE + chipRun.state = 'full' + AND ((camRun.state = 'new' AND camProcessedExp.cam_id IS NULL) + OR camRun.state = 'update') + AND camMask.label IS NULL + ) as Foo Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_find_pendingimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_find_pendingimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_find_pendingimfile.sql (revision 22161) @@ -0,0 +1,40 @@ +SELECT DISTINCT * FROM + -- the subselect is so where criteria can be specified without knowing + -- which table the field came from + (SELECT + camRun.*, + chipProcessedImfile.exp_id, + chipProcessedImfile.class_id, + chipProcessedImfile.uri, + chipProcessedImfile.bg, + chipProcessedImfile.bg_stdev, + chipProcessedImfile.bg_mean_stdev, + chipProcessedImfile.fringe_0, + chipProcessedImfile.fringe_1, + chipProcessedImfile.fringe_2, + chipProcessedImfile.ap_resid, + chipProcessedImfile.ap_resid_stdev, + chipProcessedImfile.sigma_ra, + chipProcessedImfile.sigma_dec, + chipProcessedImfile.n_stars, + chipProcessedImfile.n_extended, + chipProcessedImfile.n_cr, + chipProcessedImfile.n_astrom, + chipProcessedImfile.path_base, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel + FROM camRun + JOIN chipRun + USING(chip_id) + JOIN chipProcessedImfile + USING(chip_id) + JOIN rawExp + ON chipProcessedImfile.exp_id = rawExp.exp_id + LEFT JOIN camProcessedExp + USING(cam_id) + LEFT JOIN camMask + ON camRun.label = camMask.label + WHERE + camMask.label IS NULL) as foo Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_find_processedexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_find_processedexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_find_processedexp.sql (revision 22161) @@ -0,0 +1,18 @@ +SELECT DISTINCT + camProcessedExp.*, + rawExp.exp_tag, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel +FROM camRun +JOIN camProcessedExp + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + USING(chip_id) +JOIN rawExp + ON chipProcessedImfile.exp_id = rawExp.exp_id +WHERE -- bogus condition so there is a pre-existing where to append to + camProcessedExp.cam_id IS NOT NULL Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_pendingcleanupexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_pendingcleanupexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_pendingcleanupexp.sql (revision 22161) @@ -0,0 +1,16 @@ +SELECT + camProcessedExp.*, + camRun.state, + camRun.workdir, + camRun.label, + camRun.reduction, + camRun.expgroup, + camRun.dvodb, + camRun.tess_id, + camRun.end_stage +FROM camRun +JOIN camProcessedExp + USING(cam_id) +WHERE + (camRun.state = 'goto_cleaned' OR camRun.state = 'goto_purged') + Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_pendingcleanuprun.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_pendingcleanuprun.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_pendingcleanuprun.sql (revision 22161) @@ -0,0 +1,11 @@ +SELECT + camRun.cam_id, + rawExp.camera, + camRun.state +FROM camRun +JOIN chipRun +USING (chip_id) +JOIN rawExp +USING (exp_id) +WHERE + (camRun.state = 'goto_cleaned' OR camRun.state = 'goto_purged') Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_queue_chip_id.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_queue_chip_id.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_queue_chip_id.sql (revision 22161) @@ -0,0 +1,19 @@ +-- camtool only operates on exposures so we can safely queue more then one at a +-- time without worrying about losing the track of the generated cam_id +INSERT INTO camRun + SElECT + 0, -- cam_id + chip_id, -- chip_id + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- label + '%s', -- reduction + '%s', -- expgroup + '%s', -- dvodb + '%s', -- tess_id + '%s' -- end_stage + FROM chipRun + WHERE + chipRun.state = 'full' + AND chipRun.chip_id = %lld Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_reset_faulted_runs.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_reset_faulted_runs.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_reset_faulted_runs.sql (revision 22161) @@ -0,0 +1,7 @@ +UPDATE camRun, camProcessedExp, chipRun, rawExp +SET camRun.state = 'new' +WHERE + camRun.cam_id = camProcessedExp.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND camProcessedExp.fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_revertprocessedexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_revertprocessedexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/camtool_revertprocessedexp.sql (revision 22161) @@ -0,0 +1,7 @@ +DELETE FROM camProcessedExp +USING camProcessedExp, camRun, chipRun, rawExp +WHERE + camRun.cam_id = camProcessedExp.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND camProcessedExp.fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_change_exp_state.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_change_exp_state.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_change_exp_state.sql (revision 22161) @@ -0,0 +1,15 @@ +-- change state of chipRun from goto_cleaned to cleaned or goto_purged to purged +-- when all of the consituant imfiles are in the rgith state +-- arguments are new state (cleaned or purged) chip_id and new state again for +-- the chipProcessedImfile sub query +UPDATE chipRun + SET state = '%s' + WHERE + chipRun.chip_id = %lld + AND (SELECT + COUNT(chip_id) + FROM chipProcessedImfile + WHERE + chipProcessedImfile.chip_id = chipRun.chip_id + AND data_state != '%s' + ) = 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_change_imfile_data_state.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_change_imfile_data_state.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_change_imfile_data_state.sql (revision 22161) @@ -0,0 +1,14 @@ +-- handle changes in data_state. Used for the modes tocleanedimfile and topurgedimfile +-- args are new data_state, chip_id, class_id, and current expected state for chipRun +UPDATE chipProcessedImfile + SET + data_state = '%s' +WHERE + chip_id = %lld + AND class_id = '%s' + -- only update if chipRun.state has the expected value + AND ( + SELECT state from chipRun where chipRun.chip_id = chipProcessedImfile.chip_id + ) = '%s' + + Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_completely_processed_exp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_completely_processed_exp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_completely_processed_exp.sql (revision 22161) @@ -0,0 +1,34 @@ +-- the output of this query must match the format of chipRun row +SELECT DISTINCT + chip_id, + exp_id, + state, + workdir, + workdir_state, + label, + reduction, + expgroup, + dvodb, + tess_id, + end_stage +FROM + (SELECT + chipRun.*, + rawImfile.class_id as rawimfile_class_id, + chipProcessedImfile.class_id + FROM chipRun + JOIN rawImfile + USING(exp_id) + LEFT JOIN chipProcessedImfile + ON chipRun.chip_id = chipProcessedImfile.chip_id + AND rawImfile.exp_id = chipProcessedImfile.exp_id + AND rawImfile.class_id = chipProcessedImfile.class_id + WHERE + chipRun.state = 'new' + GROUP BY + chipRun.chip_id, + chipRun.exp_id + HAVING + COUNT(rawImfile.class_id) = COUNT(chipProcessedImfile.class_id) + AND SUM(chipProcessedImfile.fault) = 0 + ) as Foo Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_donecleanup.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_donecleanup.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_donecleanup.sql (revision 22161) @@ -0,0 +1,5 @@ +SELECT + chipRun.* +FROM chipRun +WHERE + chipRun.state = 'cleaned' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_find_rawexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_find_rawexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_find_rawexp.sql (revision 22161) @@ -0,0 +1,7 @@ +-- this query is used to find potental rawExps to be queued for chiptool +-- processeing +SELECT + * +FROM rawExp +WHERE + rawExp.fault = 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_find_unprocessed_imfile.pl =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_find_unprocessed_imfile.pl (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_find_unprocessed_imfile.pl (revision 22161) @@ -0,0 +1,8 @@ +SELECT DISTINCT + chipInputImfile.* +FROM chipInputImfile +LEFT JOIN chipProcessedImfile + USING(chip_id, class_id) +WHERE + chipProcessedImfile.chip_id IS NULL + AND chipProcessedImfile.class_id IS NULL Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_pendingcleanupimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_pendingcleanupimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_pendingcleanupimfile.sql (revision 22161) @@ -0,0 +1,17 @@ +SELECT + chipProcessedImfile.*, + chipRun.state, + chipRun.workdir, + chipRun.label, + chipRun.reduction, + chipRun.expgroup, + chipRun.dvodb, + chipRun.tess_id, + chipRun.end_stage +FROM chipRun +JOIN chipProcessedImfile + USING(chip_id) +WHERE + ((chipRun.state = 'goto_cleaned' AND chipProcessedImfile.data_state = 'full') +OR + (chipRun.state = 'goto_purged' AND chipProcessedImfile.data_state != 'purged')) Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_pendingcleanuprun.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_pendingcleanuprun.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_pendingcleanuprun.sql (revision 22161) @@ -0,0 +1,9 @@ +SELECT + chipRun.chip_id, + rawExp.camera, + chipRun.state +FROM chipRun +JOIN rawExp +USING (exp_id) +WHERE + (chipRun.state = 'goto_cleaned' OR chipRun.state = 'goto_purged') Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_pendingimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_pendingimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_pendingimfile.sql (revision 22161) @@ -0,0 +1,29 @@ +SELECT + chipRun.*, + rawImfile.class_id, + rawImfile.uri, + rawExp.exp_tag, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel +FROM chipRun +JOIN rawExp + USING(exp_id) +JOIN rawImfile + USING(exp_id) +LEFT JOIN chipProcessedImfile + ON chipRun.chip_id = chipProcessedImfile.chip_id + AND rawImfile.exp_id = chipProcessedImfile.exp_id + AND rawImfile.class_id = chipProcessedImfile.class_id +LEFT JOIN chipMask + ON chipRun.label = chipMask.label +WHERE + ((chipRun.state = 'new' + AND chipProcessedImfile.chip_id IS NULL + AND chipProcessedImfile.exp_id IS NULL + AND chipProcessedImfile.class_id IS NULL + AND chipMask.label IS NULL) + OR + (chipRun.state = 'update' + AND chipProcessedImfile.data_state = 'cleaned')) Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_processedimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_processedimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_processedimfile.sql (revision 22161) @@ -0,0 +1,23 @@ +SELECT DISTINCT + chipProcessedImfile.class_id, + chipProcessedImfile.uri, + chipProcessedImfile.bg, + chipProcessedImfile.bg_stdev, + chipProcessedImfile.bg_mean_stdev, + chipProcessedImfile.path_base, + chipRun.state, + rawExp.exp_id, + rawExp.exp_tag, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel +FROM chipRun +JOIN chipProcessedImfile + USING(chip_id) +JOIN rawExp + ON chipProcessedImfile.exp_id = rawExp.exp_id +WHERE +-- bogus test; just here so there there is a 'WHERE' stmt to append conditionals too + chipProcessedImfile.exp_id is NOT NULL + Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_revertprocessedimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_revertprocessedimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_revertprocessedimfile.sql (revision 22161) @@ -0,0 +1,5 @@ +DELETE FROM chipProcessedImfile +USING chipProcessedImfile, chipRun, rawExp +WHERE + rawExp.exp_id = chipProcessedImfile.exp_id + AND chipProcessedImfile.fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_run.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_run.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_run.sql (revision 22161) @@ -0,0 +1,3 @@ +SELECT + chipRun.* +FROM chipRun Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_unmasked.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_unmasked.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/chiptool_unmasked.sql (revision 22161) @@ -0,0 +1,43 @@ +-- SELECT +-- * +-- FROM +-- (SELECT +-- chipMask.label as label, +-- count(chipRun.chip_id) as n_chipruns +-- FROM +-- chipMask +-- LEFT JOIN +-- chipRun +-- USING +-- (label) +-- WHERE chipRun.chip_id IS NOT NULL +-- GROUP BY label) as chipMask + +-- SELECT +-- * +-- FROM +-- SELECT +-- chipRun.*, +-- rawExp.*, +-- chipRun.chip_id as n_chipruns +-- FROM +-- chipRun +-- JOIN rawExp ON chipRun.exp_id = rawExp.exp_id +-- LEFT JOIN chipMask +-- USING (label) +-- WHERE chipMask.label IS NULL +-- AND filter = 'r' +-- AND chipRun.exp_id = 61 +-- GROUP BY chipRun.label) as chipUnmask + +SELECT + * +FROM + (SELECT + chipRun.label, + count(chipRun.chip_id) as n_chipruns + FROM chipRun + LEFT JOIN chipMask + USING (label) + WHERE chipMask.label IS NULL + GROUP BY chipRun.label) as chipUnmask Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/detselect_search.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/detselect_search.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/detselect_search.sql (revision 22161) @@ -0,0 +1,29 @@ +-- this query needs to use the same fields in both of the tables in +-- the union statement, but we need to report the +-- detRunSummary.iteration in the first case, and this is missing in +-- the second case. + +SELECT DISTINCT + det_id, + good_iteration as iteration, + filelevel +FROM + (SELECT DISTINCT + detRun.*, + detRunSummary.iteration as good_iteration + FROM detRun + JOIN detRunSummary + USING(det_id) + WHERE + detRun.mode = 'master' + AND detRunSummary.accept = 1 + UNION + SELECT DISTINCT + detRun.*, + detRun.iteration as good_iteration + FROM detRun + WHERE + (detRun.mode = 'register' OR detRun.mode = 'correction') + ) as Foo +WHERE + (state = 'stop' OR state = 'register') Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/detselect_select.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/detselect_select.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/detselect_select.sql (revision 22161) @@ -0,0 +1,27 @@ +SELECT DISTINCT + * +FROM + (SELECT DISTINCT + detRun.state, + detNormalizedImfile.* + FROM detNormalizedImfile + JOIN detRun + USING(det_id) + JOIN detRunSummary + ON detNormalizedImfile.det_id = detRunSummary.det_id + AND detNormalizedImfile.iteration = detRunSummary.iteration + WHERE + detRun.mode = 'master' + AND detRunSummary.accept = 1 + UNION + SELECT DISTINCT + detRun.state, + detRegisteredImfile.* + FROM detRegisteredImfile + JOIN detRun + USING(det_id) + WHERE + (detRun.mode = 'register' OR detRun.mode = 'correction') + ) as Foo +WHERE + (state = 'stop' OR state = 'register') Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_addprocessedexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_addprocessedexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_addprocessedexp.sql (revision 22161) @@ -0,0 +1,34 @@ +SELECT DISTINCT + detProcessedImfile.det_id, + detRun.iteration, + detRun.det_type, + detProcessedImfile.exp_id +FROM detRun +JOIN detInputExp + ON detRun.det_id = detInputExp.det_id + AND detRun.iteration = detInputExp.iteration +JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id +JOIN detProcessedImfile + ON detInputExp.det_id = detProcessedImfile.det_id + AND detInputExp.exp_id = detProcessedImfile.exp_id +LEFT JOIN detProcessedExp + ON detInputExp.det_id = detProcessedExp.det_id + AND detProcessedImfile.exp_id= detProcessedExp.exp_id +LEFT JOIN rawImfile + ON detInputExp.exp_id = rawImfile.exp_id + AND detProcessedImfile.class_id = rawImfile.class_id +WHERE + detRun.state = 'run' + AND (detRun.mode = 'master' or detRun.mode = 'verify') + AND detProcessedExp.det_id IS NULL + AND detProcessedExp.exp_id IS NULL + AND detInputExp.include = 1 + AND detRun.det_id = %lld + AND detProcessedImfile.exp_id = %lld +GROUP BY + detProcessedImfile.class_id, + rawImfile.class_id, + detRun.det_id +HAVING + COUNT(detProcessedImfile.class_id) = COUNT(rawImfile.class_id) Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_childlessrun.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_childlessrun.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_childlessrun.sql (revision 22161) @@ -0,0 +1,12 @@ +-- dettool -childlessrun + +SELECT DISTINCT + detRun.* + FROM detRun + LEFT JOIN detRun as foo + ON foo.ref_det_id = detRun.det_id +-- XXX do we need to restrict to foo.ref_iter = detRun.iteration ? + WHERE + detRun.state = 'stop' + AND detRun.mode = 'master' + AND foo.det_id IS NULL Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_definebydetrun.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_definebydetrun.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_definebydetrun.sql (revision 22161) @@ -0,0 +1,12 @@ +-- dettool -definebydetrun + +INSERT INTO detInputExp + SELECT + %lld, + 0, + detResidExp.exp_id, + detResidExp.accept + FROM detResidExp + JOIN rawExp + USING(exp_id) + WHERE det_id = %lld Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_detrunsummary.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_detrunsummary.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_detrunsummary.sql (revision 22161) @@ -0,0 +1,9 @@ +SELECT DISTINCT + detRunSummary.*, + detRun.det_type, + detRun.mode + FROM detRun + JOIN detRunSummary + USING(det_id, iteration) + WHERE + detRun.state = 'run' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_find_completed_runs.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_find_completed_runs.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_find_completed_runs.sql (revision 22161) @@ -0,0 +1,26 @@ +SELECT DISTINCT + det_id, + iteration +FROM + (SELECT DISTINCT + detRun.det_id, + detRun.iteration, + detInputExp.exp_id + FROM detRun + LEFT JOIN detInputExp + ON detRun.det_id = detInputExp.det_id + AND detRun.iteration = detInputExp.iteration + LEFT JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id + LEFT JOIN detResidExp + ON detRun.det_id = detResidExp.det_id + AND detRun.iteration = detResidExp.iteration + AND detInputExp.exp_id = detResidExp.exp_id + WHERE + detRun.state = 'run' + GROUP BY + detRun.det_id, + detRun.iteration + HAVING + COUNT(detResidExp.exp_id) = COUNT(detInputExp.exp_id) + ) AS residdetrun Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_input.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_input.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_input.sql (revision 22161) @@ -0,0 +1,4 @@ +SELECT DISTINCT * + FROM detInputExp + JOIN rawExp + USING(exp_id) Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_normalizedexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_normalizedexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_normalizedexp.sql (revision 22161) @@ -0,0 +1,8 @@ +SELECT + detNormalizedExp.* + FROM detNormalizedExp + JOIN detRun + USING(det_id, iteration) + WHERE + detRun.state = 'run' + AND detRun.mode = 'master' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_normalizedimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_normalizedimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_normalizedimfile.sql (revision 22161) @@ -0,0 +1,8 @@ +SELECT + detNormalizedImfile.* + FROM detNormalizedImfile + JOIN detRun + USING(det_id, iteration) + WHERE + detRun.state = 'run' + AND detRun.mode = 'master' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_normalizedstat.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_normalizedstat.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_normalizedstat.sql (revision 22161) @@ -0,0 +1,12 @@ +SELECT + * +FROM + (SELECT + detNormalizedStatImfile.* + FROM detNormalizedStatImfile + JOIN detRun + USING(det_id, iteration) + WHERE + detRun.state = 'run' + AND detRun.mode = 'master' + ) as Foo Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pending.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pending.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pending.sql (revision 22161) @@ -0,0 +1,10 @@ +-- dettool -pending : not sure this is still used (EAM : 2008.07.10) + +SELECT + rawExp.* + FROM rawExp + LEFT JOIN detInputExp + ON rawExp.exp_id = detInputExp.exp_id + WHERE + detInputExp.exp_id IS NULL + AND rawExp.object != 'object' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_normalizedexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_normalizedexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_normalizedexp.sql (revision 22161) @@ -0,0 +1,9 @@ +SELECT + detNormalizedExp.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detNormalizedExp + USING(det_id,iteration) +WHERE + detRunSummary.data_state = 'goto_cleaned' +AND detNormalizedExp.data_state = 'full' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_normalizedimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_normalizedimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_normalizedimfile.sql (revision 22161) @@ -0,0 +1,9 @@ +SELECT + detNormalizedImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detNormalizedImfile + USING(det_id,iteration) +WHERE + detRunSummary.data_state = 'goto_cleaned' +AND detNormalizedImfile.data_state = 'full' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_normalizedstat.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_normalizedstat.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_normalizedstat.sql (revision 22161) @@ -0,0 +1,9 @@ +SELECT + detNormalizedStatImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detNormalizedStatImfile + USING(det_id,iteration) +WHERE + detRunSummary.data_state = 'goto_cleaned' +AND detNormalizedStatImfile.data_state = 'full' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_processedexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_processedexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_processedexp.sql (revision 22161) @@ -0,0 +1,10 @@ +-- need to restrict to a single detRunSummary (require all to say 'cleaned'?) +SELECT + detProcessedExp.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detProcessedExp + USING(det_id) +WHERE + detRunSummary.data_state = 'goto_cleaned' +AND detProcessedExp.data_state = 'full' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_processedimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_processedimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_processedimfile.sql (revision 22161) @@ -0,0 +1,10 @@ +-- need to restrict to a single detRunSummary (require all to say 'cleaned'?) +SELECT + detProcessedImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detProcessedImfile + USING(det_id) +WHERE + detRunSummary.data_state = 'goto_cleaned' +AND detProcessedImfile.data_state = 'full' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_residexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_residexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_residexp.sql (revision 22161) @@ -0,0 +1,9 @@ +SELECT + detResidExp.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detResidExp + USING(det_id,iteration) +WHERE + detRunSummary.data_state = 'goto_cleaned' +AND detResidExp.data_state = 'full' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_residimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_residimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_residimfile.sql (revision 22161) @@ -0,0 +1,9 @@ +SELECT + detResidImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detResidImfile + USING(det_id,iteration) +WHERE + detRunSummary.data_state = 'goto_cleaned' +AND detResidImfile.data_state = 'full' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_stacked.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_stacked.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_pendingcleanup_stacked.sql (revision 22161) @@ -0,0 +1,9 @@ +SELECT + detStackedImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detStackedImfile + USING(det_id,iteration) +WHERE + detRunSummary.data_state = 'goto_cleaned' +AND detStackedImfile.data_state = 'full' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_processedimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_processedimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_processedimfile.sql (revision 22161) @@ -0,0 +1,13 @@ +SELECT DISTINCT + detRun.det_type, + rawExp.exp_time, + detProcessedImfile.* +FROM detProcessedImfile +JOIN detRun + USING(det_id) +JOIN detInputExp + ON detRun.det_id = detInputExp.det_id + AND detRun.iteration = detInputExp.iteration + AND detProcessedImfile.exp_id = detInputExp.exp_id +JOIN rawExp + ON rawExp.exp_id = detProcessedImfile.exp_id Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_raw.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_raw.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_raw.sql (revision 22161) @@ -0,0 +1,12 @@ +SELECT + detInputExp.det_id, + detRun.det_type, + rawImfile.* +FROM detRun +JOIN detInputExp + USING(det_id) +JOIN rawImfile + ON detInputExp.exp_id = rawImfile.exp_id +WHERE + detRun.state = 'run' + AND detRun.mode = 'master' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_residexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_residexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_residexp.sql (revision 22161) @@ -0,0 +1,11 @@ +SELECT + detRun.mode, + detResidExp.*, + detInputExp.include + FROM detRun + JOIN detInputExp + USING(det_id, iteration) + JOIN detResidExp + USING(det_id, iteration, exp_id) + WHERE + detRun.state = 'run' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_residimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_residimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_residimfile.sql (revision 22161) @@ -0,0 +1,11 @@ +SELECT + detRun.det_type, + detRun.mode, + detResidImfile.*, + rawExp.exp_time + FROM detResidImfile + JOIN detRun + USING(det_id, iteration) + JOIN rawExp + USING(exp_id) + WHERE Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertdetrunsummary.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertdetrunsummary.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertdetrunsummary.sql (revision 22161) @@ -0,0 +1,3 @@ +DELETE FROM detRunSummary +WHERE + fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertnormalizedexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertnormalizedexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertnormalizedexp.sql (revision 22161) @@ -0,0 +1,3 @@ +DELETE FROM detNormalizedExp +WHERE + fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertnormalizedimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertnormalizedimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertnormalizedimfile.sql (revision 22161) @@ -0,0 +1,3 @@ +DELETE FROM detNormalizedImfile +WHERE + fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertnormalizedstat.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertnormalizedstat.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertnormalizedstat.sql (revision 22161) @@ -0,0 +1,3 @@ +DELETE FROM detNormalizedStatImfile +WHERE + fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertprocessedexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertprocessedexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertprocessedexp.sql (revision 22161) @@ -0,0 +1,3 @@ +DELETE FROM detProcessedExp +WHERE + fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertprocessedimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertprocessedimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertprocessedimfile.sql (revision 22161) @@ -0,0 +1,3 @@ +DELETE FROM detProcessedImfile +WHERE + fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertresidexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertresidexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertresidexp.sql (revision 22161) @@ -0,0 +1,3 @@ +DELETE FROM detResidExp +WHERE + fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertresidimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertresidimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertresidimfile.sql (revision 22161) @@ -0,0 +1,3 @@ +DELETE FROM detResidImfile +WHERE + fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertstacked.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertstacked.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_revertstacked.sql (revision 22161) @@ -0,0 +1,3 @@ +DELETE FROM detStackedImfile +WHERE + fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_runs.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_runs.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_runs.sql (revision 22161) @@ -0,0 +1,2 @@ + +SELECT * from detRun Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_stacked.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_stacked.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_stacked.sql (revision 22161) @@ -0,0 +1,12 @@ +-- select detStackedImfile.* +-- by: +-- where det_id, iteration, class_id is not in detNormalizedImfile + +SELECT + detStackedImfile.* + FROM detStackedImfile + JOIN detRun + USING(det_id, iteration) + WHERE + detRun.state = 'run' + AND detRun.mode = 'master' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_start_new_iteration.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_start_new_iteration.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_start_new_iteration.sql (revision 22161) @@ -0,0 +1,33 @@ +-- select detRun.iteration +-- select detInputExp.exp_id +-- select detResidExp.accept +-- by: +-- find the current iteration bassed on det_id +-- find all exp_ids in the current det_id/iteration from detInputExp +-- find all exp_ids in the current det_id/iteration from detResidExp +-- compare the counts of exp_ids + +SELECT * FROM + (SELECT DISTINCT + detRun.det_id AS det_id, + detRun.iteration, + detInputExp.exp_id, + detResidExp.accept + FROM detRun + JOIN detInputExp + ON detRun.det_id = detInputExp.det_id + AND detRun.iteration = detInputExp.iteration + JOIN detResidExp + ON detRun.det_id = detResidExp.det_id + AND detRun.iteration = detResidExp.iteration + AND detInputExp.exp_id = detResidExp.exp_id + WHERE + detRun.state = 'run' + AND detRun.mode = 'master' + GROUP BY + detRun.det_id, + detRun.iteration, + detInputExp.exp_id + HAVING + COUNT(detResidExp.exp_id) = COUNT(detInputExp.exp_id) + ) as Foo Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_stop_completed_correct_runs.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_stop_completed_correct_runs.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_stop_completed_correct_runs.sql (revision 22161) @@ -0,0 +1,29 @@ +UPDATE detRun +SET state = 'stop' +WHERE det_id = (SELECT det_id FROM + (SELECT + det1.det_id, + imfile.class_id, + detRegisteredImfile.class_id as reg_class_id + FROM detRun AS det1 + INNER JOIN detRun as det2 + ON det1.ref_det_run = det2.det_id + -- do we need to restrict by iteration? not clear this logic below is right or tested + JOIN detNormalizedImfile as imfile + ON det2.det_id = imfile.det_id + AND det2.iteration = imfile.iteration + LEFT JOIN detRegisteredImfile + ON det1.det_id = detRegisteredImfile.det_id + AND det1.iteration = detRegisteredImfile.iteration + AND imfile.class_id = detRegisteredImfile.class_id + WHERE + det1.state = 'run' + AND det1.mode = 'correction' + AND det2.state = 'stop' + GROUP BY + det1.det_id + HAVING + COUNT(imfile.class_id) = COUNT(reg_class_id) + AND SUM(imfile.fault) = 0 + ) as Foo +) Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_tocorrectexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_tocorrectexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_tocorrectexp.sql (revision 22161) @@ -0,0 +1,4 @@ +SELECT * FROM detRun +WHERE + detRun.mode = 'correction' + AND detRun.state = 'run' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_tocorrectimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_tocorrectimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_tocorrectimfile.sql (revision 22161) @@ -0,0 +1,32 @@ +-- select imfiles from detRun 1 (the reference detRun) that match the desired correction in detRun 2 +SELECT + det1.*, + imfile.class_id, + imfile.uri, + imfile.bg, + imfile.bg_stdev, + imfile.bg_mean_stdev, + imfile.user_1, + imfile.user_2, + imfile.user_3, + imfile.user_4, + imfile.user_5, + imfile.path_base, + imfile.fault +FROM detRun AS det1 +JOIN detRun as det2 + ON det1.ref_det_id = det2.det_id +JOIN detNormalizedImfile as imfile + ON det1.ref_det_id = imfile.det_id + AND det1.ref_iter = imfile.iteration +LEFT JOIN detRegisteredImfile + ON det1.det_id = detRegisteredImfile.det_id + AND det1.iteration = detRegisteredImfile.iteration + AND imfile.class_id = detRegisteredImfile.class_id +WHERE + det1.state = 'run' + AND det1.mode = 'correction' + AND det2.state = 'stop' + AND detRegisteredImfile.det_id IS NULL + AND detRegisteredImfile.iteration IS NULL + AND detRegisteredImfile.class_id IS NULL Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_todetrunsummary.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_todetrunsummary.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_todetrunsummary.sql (revision 22161) @@ -0,0 +1,47 @@ +-- select detRun.det_id +-- select detRun.iteration +-- select detRun.det_type +-- by: +-- find the current iteration bassed on det_id +-- find all exp_ids in the current det_id/iteration from detInputExp +-- find all exp_ids in the current det_id/iteration from detResidExp +-- compare the counts of exp_ids + +SELECT DISTINCT + det_id, + iteration, + det_type, + mode, + workdir, + camera +FROM + (SELECT DISTINCT + detRun.det_id, + detRun.iteration, + detRun.det_type, + detRun.mode, + detRun.workdir, + detInputExp.exp_id, + rawExp.camera + FROM detRun + JOIN detInputExp + USING(det_id, iteration) + JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id + LEFT JOIN detResidExp + ON detRun.det_id = detResidExp.det_id + AND detRun.iteration = detResidExp.iteration + AND detInputExp.exp_id = detResidExp.exp_id + LEFT JOIN detRunSummary + ON detRun.det_id = detRunSummary.det_id + AND detRun.iteration = detRunSummary.iteration + WHERE + detRun.state = 'run' + AND detRunSummary.det_id IS NULL + AND detRunSummary.iteration IS NULL + GROUP BY + detRun.det_id, + detRun.iteration + HAVING + COUNT(detResidExp.exp_id) = COUNT(detInputExp.exp_id) + ) AS Foo Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_tonormalize.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_tonormalize.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_tonormalize.sql (revision 22161) @@ -0,0 +1,27 @@ +SELECT DISTINCT + detRun.det_type, + detRun.workdir, + rawExp.camera, + detStackedImfile.uri, + detNormalizedStatImfile.* + FROM detRun + JOIN detStackedImfile + USING(det_id, iteration) + JOIN detInputExp + USING(det_id, iteration) + JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id + JOIN detNormalizedStatImfile + ON detStackedImfile.det_id = detNormalizedStatImfile.det_id + AND detStackedImfile.iteration = detNormalizedStatImfile.iteration + AND detStackedImfile.class_id = detNormalizedStatImfile.class_id + LEFT JOIN detNormalizedImfile + ON detNormalizedStatImfile.det_id = detNormalizedImfile.det_id + AND detNormalizedStatImfile.iteration = detNormalizedImfile.iteration + AND detNormalizedStatImfile.class_id = detNormalizedImfile.class_id + WHERE + detNormalizedImfile.det_id IS NULL + AND detNormalizedImfile.iteration IS NULL + AND detNormalizedImfile.class_id IS NULL + AND detNormalizedStatImfile.fault = 0 + AND detRun.state = 'run' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_tonormalizedexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_tonormalizedexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_tonormalizedexp.sql (revision 22161) @@ -0,0 +1,48 @@ +-- this is used by both 'tonormalizedexp' and 'addnormalizedexp' + +SELECT + * +FROM (SELECT + * + FROM ( + SELECT DISTINCT + detRun.det_id, + detRun.iteration, + detRun.det_type, + detRun.workdir, + rawExp.camera, + rawExp.telescope, + rawExp.exp_type, + detNormalizedImfile.class_id as norm_class_id, + detNormalizedImfile.fault, + detStackedImfile.class_id as stack_class_id + FROM detRun + JOIN detInputExp + ON detRun.det_id = detInputExp.det_id + AND detRun.iteration = detInputExp.iteration + JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id + JOIN detStackedImfile + ON detStackedImfile.det_id = detRun.det_id + AND detStackedImfile.iteration = detRun.iteration + LEFT JOIN detNormalizedImfile + ON detStackedImfile.det_id = detNormalizedImfile.det_id + AND detStackedImfile.iteration = detNormalizedImfile.iteration + AND detStackedImfile.class_id = detNormalizedImfile.class_id + LEFT JOIN detNormalizedExp + ON detInputExp.det_id = detNormalizedExp.det_id + AND detInputExp.iteration = detNormalizedExp.iteration + WHERE + detRun.state = 'run' AND + detRun.mode = 'master' AND + detNormalizedExp.det_id IS NULL AND + detNormalizedExp.iteration IS NULL AND + detInputExp.include = 1 + ) as Foo + GROUP BY + det_id, + iteration + HAVING + COUNT(norm_class_id) = COUNT(stack_class_id) + AND SUM(fault) = 0 +) as Bar Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_tonormalizedstat.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_tonormalizedstat.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_tonormalizedstat.sql (revision 22161) @@ -0,0 +1,58 @@ +-- select detRun.det_id (det_id) +-- select detRun.iteration +-- by: +-- find the current iteration bassed on det_id +-- find all exp_ids in the current det_id/iteration from detInputExp +-- sort to detInputExp.imfiles to find the largest value per det_id/iter +-- compare imfiles to the number of detStackedImfiles by class_id +-- and: +-- ??? +-- det_id is not in detStackedImfile; +-- iteration is not in detStackedImfile; + +SELECT + det_id, + det_type, + iteration, + camera, + workdir, + class_id +FROM + (SELECT DISTINCT + detRun.det_id, + detRun.det_type, + detRun.iteration, + detRun.workdir, + rawExp.camera, + detStackedImfile.class_id, + rawImfile.class_id as rawimfile_class_id + FROM detRun + JOIN detInputExp + ON detRun.det_id = detInputExp.det_id + AND detRun.iteration = detInputExp.iteration + JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id + JOIN rawImfile + ON rawExp.exp_id = rawImfile.exp_id + LEFT JOIN detStackedImfile + ON detInputExp.det_id = detStackedImfile.det_id + AND detInputExp.iteration = detStackedImfile.iteration + AND rawImfile.class_id = detStackedImfile.class_id + LEFT JOIN detNormalizedStatImfile + ON detStackedImfile.det_id = detNormalizedStatImfile.det_id + AND detStackedImfile.iteration = detNormalizedStatImfile.iteration + AND detStackedImfile.class_id = detNormalizedStatImfile.class_id + WHERE + detRun.state = 'run' + AND detRun.mode = 'master' + AND detNormalizedStatImfile.det_id IS NULL + AND detNormalizedStatImfile.iteration IS NULL + AND detNormalizedStatImfile.class_id IS NULL + GROUP BY +-- rawExp.exp_id, + detRun.iteration, + detRun.det_id + HAVING + COUNT(rawImfile.class_id) = COUNT(detStackedImfile.class_id) + AND SUM(detStackedImfile.fault) = 0 + ) as tonormalizedstat Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_toprocessedexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_toprocessedexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_toprocessedexp.sql (revision 22161) @@ -0,0 +1,61 @@ +-- select detProcessedImfile.det_id +-- select detRun.iteration +-- select detRun.det_type +-- select detProcessedImfile.exp_id +-- by: +-- find the current iteration bassed on det_id +-- find all exp_ids in the current det_id/iteration from detInputExp +-- find all rawImfiles in the current exp_ids +-- compare to detProcessedImfiles by det_id/exp_id +-- found how many imfile there are in each class_id +-- and: +-- det_id is not in detProcessedExp; +-- iteration is not in detProcessedExp; + +SELECT DISTINCT + det_id, + iteration, + det_type, + exp_id, + camera, + workdir, + exp_tag +FROM + (SELECT DISTINCT + detRun.det_id, + detRun.iteration, + detRun.det_type, + detRun.workdir, + detProcessedImfile.exp_id, + detProcessedImfile.class_id, + rawImfile.class_id as rawimfile_class_id, + rawExp.camera, + rawExp.exp_tag + FROM detRun + JOIN detInputExp + USING(det_id, iteration) + JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id + JOIN rawImfile + ON rawExp.exp_id = rawImfile.exp_id + LEFT JOIN detProcessedImfile + ON detRun.det_id = detProcessedImfile.det_id + AND detInputExp.exp_id = detProcessedImfile.exp_id + AND rawImfile.class_id = detProcessedImfile.class_id + LEFT JOIN detProcessedExp + ON detProcessedImfile.det_id = detProcessedExp.det_id + AND detProcessedImfile.exp_id = detProcessedExp.exp_id + WHERE + detRun.state = 'run' + AND (detRun.mode = 'master' or detRun.mode = 'verify') + AND detProcessedExp.det_id IS NULL + AND detProcessedExp.exp_id IS NULL + AND detInputExp.include = 1 + GROUP BY + rawExp.exp_id, + detRun.det_id + HAVING + COUNT(detProcessedImfile.class_id) = COUNT(rawImfile.class_id) + AND SUM(detProcessedImfile.fault) = 0 + ) AS detProcessedExp + Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_toprocessedimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_toprocessedimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_toprocessedimfile.sql (revision 22161) @@ -0,0 +1,24 @@ +SELECT + detRun.det_id, + detRun.det_type, + detRun.workdir, + detRun.reduction, + rawImfile.*, + rawExp.exp_tag +FROM detRun +JOIN detInputExp + USING(det_id, iteration) +JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id +JOIN rawImfile + ON detInputExp.exp_id = rawImfile.exp_id +LEFT JOIN detProcessedImfile + ON detInputExp.det_id = detProcessedImfile.det_id + AND rawImfile.exp_id = detProcessedImfile.exp_id + AND rawImfile.class_id = detProcessedImfile.class_id +WHERE + detRun.state = 'run' + AND (detRun.mode = 'master' or detRun.mode = 'verify') + AND detProcessedImfile.det_id IS NULL + AND detProcessedImfile.exp_id IS NULL + AND detProcessedImfile.class_id IS NULL Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_toresidexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_toresidexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_toresidexp.sql (revision 22161) @@ -0,0 +1,70 @@ +-- this is used by both 'toresidexp' and 'addresidexp' + +-- which returns a list of exposures for which all component class ids have had +-- residuals created. This list includes the detrend id, iteration, exposure +-- id, detrend type, and whether the exposure was included in the stack for +-- this iteration. + +-- select detRun.det_id +-- select detRun.iteration +-- select detRun.det_type +-- select detInputExp.exp_id +-- select detInputExp.include +-- by: +-- find the current iteration bassed on det_id +-- find all exp_ids in the current det_id/iteration from detInputExp +-- compare to detInputExp.imfiles to derResidImfile by class_id +-- and: +-- detResidImfile.{det_id, iteration, exp_id} is not in detResidExp + +SELECT DISTINCT + det_id, + iteration, + det_type, + mode, + exp_id, + include, + camera, + workdir, + exp_tag +FROM + (SELECT DISTINCT + detRun.det_id AS det_id, + detRun.iteration, + detRun.det_type, + detRun.mode, + detRun.workdir, + detInputExp.exp_id, + detInputExp.include, + rawExp.camera, + rawExp.exp_tag, + detResidImfile.class_id + FROM detRun + JOIN detInputExp + USING(det_id, iteration) + JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id + JOIN rawImfile + on rawExp.exp_id = rawImfile.exp_id + LEFT JOIN detResidImfile + ON detRun.det_id = detResidImfile.det_id + AND detRun.iteration = detResidImfile.iteration + AND detInputExp.exp_id = detResidImfile.exp_id + AND rawImfile.class_id = detResidImfile.class_id + LEFT JOIN detResidExp + ON detResidImfile.det_id = detResidExp.det_id + AND detResidImfile.iteration = detResidExp.iteration + AND detResidImfile.exp_id = detResidExp.exp_id + WHERE + detRun.state = 'run' + AND detResidExp.det_id IS NULL + AND detResidExp.iteration IS NULL + AND detResidExp.exp_id IS NULL + GROUP BY + detInputExp.exp_id, + detRun.iteration, + detRun.det_id + HAVING + COUNT(rawImfile.class_id) = COUNT(detResidImfile.class_id) + AND SUM(detResidImfile.fault = 0) + ) AS toresidexp Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_toresidimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_toresidimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_toresidimfile.sql (revision 22161) @@ -0,0 +1,94 @@ +-- which returns a list of processed imfiles (with corresponding detrend id, +-- iteration, class id, uri, type) for which the appropriate stacked imfile has +-- been normalised, and which have not been masked out by detResidImfileAnalysis + +SELECT DISTINCT + detRun.det_id, + detRun.iteration, + detRun.det_type, + detRun.mode, + detRun.workdir, + detRun.reduction, + detProcessedImfile.exp_id, + detProcessedImfile.class_id, + detProcessedImfile.uri, + detNormalizedImfile.uri AS det_uri, + detRun.det_id AS ref_det_id, + detRun.iteration AS ref_iter, + rawExp.camera, + rawExp.exp_tag +FROM detRun +JOIN detInputExp + USING(det_id, iteration) +JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id +JOIN detProcessedImfile + ON detRun.det_id = detProcessedImfile.det_id + AND detInputExp.exp_id = detProcessedImfile.exp_id +JOIN detNormalizedImfile + ON detRun.det_id = detNormalizedImfile.det_id + AND detRun.iteration = detNormalizedImfile.iteration + AND detProcessedImfile.class_id = detNormalizedImfile.class_id +JOIN detNormalizedExp + ON detRun.det_id = detNormalizedExp.det_id + AND detRun.iteration = detNormalizedExp.iteration +LEFT JOIN detResidImfile + ON detRun.det_id = detResidImfile.det_id + AND detRun.iteration = detResidImfile.iteration + AND detProcessedImfile.exp_id = detResidImfile.exp_id + AND detProcessedImfile.class_id = detResidImfile.class_id +WHERE + detRun.state = 'run' + AND detRun.mode = 'master' + AND detNormalizedImfile.fault = 0 + AND detNormalizedExp.fault = 0 + AND detResidImfile.det_id IS NULL + AND detResidImfile.iteration IS NULL + AND detResidImfile.exp_id IS NULL + AND detResidImfile.class_id IS NULL +UNION +SELECT + detRun.det_id, + detRun.iteration, + detRun.det_type, + detRun.mode, + detRun.workdir, + detRun.reduction, + detProcessedImfile.exp_id, + detProcessedImfile.class_id, + detProcessedImfile.uri, + detNormalizedImfile.uri AS det_uri, + detRun.ref_det_id, + detRun.ref_iter, + rawExp.camera, + rawExp.exp_tag +FROM detRun +JOIN detRun AS detRunRef + ON detRunRef.det_id = detRun.ref_det_id +JOIN detInputExp + ON detInputExp.det_id = detRun.det_id + AND detInputExp.iteration = detRun.iteration +JOIN rawExp + ON rawExp.exp_id = detInputExp.exp_id +JOIN detProcessedImfile + ON detProcessedImfile.det_id = detRun.det_id + AND detProcessedImfile.exp_id = detInputExp.exp_id +JOIN detNormalizedImfile + ON detNormalizedImfile.det_id = detRun.ref_det_id + AND detNormalizedImfile.iteration = detRun.ref_iter + AND detNormalizedImfile.class_id = detProcessedImfile.class_id +LEFT JOIN detResidImfile + ON detResidImfile.det_id = detRun.det_id + AND detResidImfile.iteration = detRun.iteration + AND detResidImfile.exp_id = detProcessedImfile.exp_id + AND detResidImfile.class_id = detProcessedImfile.class_id +WHERE + detRun.state = 'run' + AND detRun.mode = 'verify' + AND detRunRef.state = 'stop' + AND detNormalizedImfile.fault = 0 + AND detResidImfile.det_id IS NULL + AND detResidImfile.iteration IS NULL + AND detResidImfile.exp_id IS NULL + AND detResidImfile.class_id IS NULL +ORDER BY exp_id Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_tostacked.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_tostacked.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/dettool_tostacked.sql (revision 22161) @@ -0,0 +1,52 @@ +-- select detRun.iteration +-- select detProcessedImfile.det_id +-- select detProcessedImfile.class_id +-- by: +-- find the current iteration bassed on det_id +-- find all exp_ids in the current det_id/iteration from detInputExp +-- find all rawImfiles in the current exp_ids +-- compare to detProcessedImfiles by det_id/exp_id +-- found how many imfile there are in each class_id +-- and: +-- det_id is not in detStackedImfile; +-- iteration is not in detStackedImfile; +-- class_id is not in detStackedImfile; + +SELECT + detProcessedImfile.det_id, + detRun.iteration, + detRun.det_type, + detRun.workdir, + detRun.reduction, + detProcessedImfile.class_id, + rawExp.camera, + rawExp.exp_tag +FROM detRun +JOIN detInputExp + ON detRun.det_id = detInputExp.det_id + AND detRun.iteration = detInputExp.iteration +JOIN rawExp + ON detInputExp.exp_id = rawExp.exp_id +JOIN rawImfile + ON detInputExp.exp_id = rawImfile.exp_id +LEFT JOIN detProcessedImfile + ON detInputExp.det_id = detProcessedImfile.det_id + AND detInputExp.exp_id = detProcessedImfile.exp_id + AND rawImfile.class_id = detProcessedImfile.class_id +LEFT JOIN detStackedImfile + ON detInputExp.det_id = detStackedImfile.det_id + AND detInputExp.iteration = detStackedImfile.iteration + AND rawImfile.class_id = detStackedImfile.class_id +WHERE + detRun.state = 'run' + AND detRun.mode = 'master' + AND detStackedImfile.det_id IS NULL + AND detStackedImfile.iteration IS NULL + AND detStackedImfile.class_id IS NULL + AND detInputExp.include = 1 +GROUP BY + rawImfile.class_id, + detRun.det_id +HAVING + COUNT(detProcessedImfile.class_id) = COUNT(rawImfile.class_id) + AND SUM(detProcessedImfile.fault) = 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_definebyquery.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_definebyquery.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_definebyquery.sql (revision 22161) @@ -0,0 +1,64 @@ +-- Get list of warps that can be diffed, with any associated diff, +-- and the best stack to use as a template +-- Warps without an existing diff can be identified by a NULL diff_id +SELECT + warpsToDiff.warp_id, + warpsToDiff.skycell_id, + warpsToDiff.tess_id, + warpsToDiff.filter, + stacksForDiff.stack_label, + warpsToDiff.warp_label, + warpsToDiff.good_frac, + warpsToDiff.diff_id, + warpsToDiff.kind, + warpsToDiff.label, + current_stack_id, + best_stack_id +FROM ( + -- Get list of warps that can be diffed, with any associated diff + SELECT + warpSkyfile.warp_id, + warpSkyfile.skycell_id, + warpSkyfile.tess_id, + warpSkyfile.good_frac, + warpRun.label, + filter, + warpRun.label as warp_label, + diffInputs.diff_id, + diffInputs.kind, + diffTemplates.stack_id AS current_stack_id + FROM warpSkyfile + JOIN warpRun USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + -- Check if it has an associated diff + LEFT JOIN diffInputSkyfile AS diffInputs + ON diffInputs.warp_id = warpSkyfile.warp_id + AND diffInputs.skycell_id = warpSkyfile.skycell_id + AND diffInputs.template = 0 -- only join input files + -- Get the stack_id currently used as a template, if any + LEFT JOIN diffInputSkyfile AS diffTemplates + ON diffTemplates.diff_id = diffInputs.diff_id + AND diffTemplates.template != 0 -- only join template files + WHERE + warpSkyfile.fault = 0 + AND warpSkyfile.ignored = 0 + ) AS warpsToDiff +-- Get best stack as a function of skycell_id, filter +JOIN ( + SELECT + MAX(stack_id) AS best_stack_id, -- most recent stack, by virtue of auto-increment + skycell_id, + tess_id, + filter, + label as stack_label + FROM stackRun + JOIN stackSumSkyfile USING(stack_id) + WHERE stackRun.state = 'full' + AND stackSumSkyfile.fault = 0 + GROUP BY + skycell_id, + filter + ) AS stacksForDiff USING(skycell_id, filter) Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_donecleanup.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_donecleanup.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_donecleanup.sql (revision 22161) @@ -0,0 +1,5 @@ +SELECT + diffRun.* +FROM diffRun +WHERE + diffRun.state = 'cleaned' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_inputskyfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_inputskyfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_inputskyfile.sql (revision 22161) @@ -0,0 +1,75 @@ +SELECT * FROM + (SELECT + diffRun.diff_id, + diffRun.skycell_id, + diffRun.tess_id, + 0 as stack_id, + warpSkyfile.warp_id, + warpSkyfile.uri, + warpSkyfile.path_base, + diffInputSkyfile.template, + rawExp.camera + FROM diffRun + JOIN diffInputSkyfile + USING(diff_id) + JOIN warpSkyfile + ON diffInputSkyfile.warp_id = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.tess_id = warpSkyfile.tess_id + JOIN warpRun + ON diffInputSkyfile.warp_id = warpRun.warp_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN chipProcessedImfile + USING(chip_id) + JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id + WHERE + diffRun.state = 'new' + AND warpRun.state = 'full' + AND fakeRun.state = 'full' + AND camRun.state = 'full' + AND chipRun.state = 'full' + -- where hook %s + UNION + SELECT + diffRun.diff_id, + diffRun.skycell_id, + diffRun.tess_id, + stackSumSkyfile.stack_id, + 0 as warp_id, + stackSumSkyfile.uri, + stackSumSkyfile.path_base, + diffInputSkyfile.template, + rawExp.camera + FROM diffRun + JOIN diffInputSkyfile + USING(diff_id) + JOIN stackSumSkyfile + ON diffInputSkyfile.stack_id = stackSumSkyfile.stack_id + JOIN stackInputSkyfile + ON diffInputSkyfile.stack_id = stackInputSkyfile.stack_id + JOIN warpRun + ON stackInputSkyfile.warp_id = warpRun.warp_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN chipProcessedImfile + USING(chip_id) + JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id + WHERE + diffRun.state = 'new' + AND warpRun.state = 'full' + AND fakeRun.state = 'full' + AND camRun.state = 'full' + AND chipRun.state = 'full' + -- where hook %s + ) as Foo Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_pendingcleanuprun.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_pendingcleanuprun.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_pendingcleanuprun.sql (revision 22161) @@ -0,0 +1,25 @@ +-- does this result in too many entries (one for each diffInputSkyfile?) +-- all of this is just to get the camera used for the diff run +SELECT + diffRun.diff_id, + rawExp.camera, + diffRun.state +FROM diffRun +JOIN diffInputSkyfile + USING(diff_id) +JOIN warpSkyfile + ON diffInputSkyfile.warp_id = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.tess_id = warpSkyfile.tess_id +JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) +WHERE + diffRun.state = 'goto_cleaned' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_pendingcleanupskyfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_pendingcleanupskyfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_pendingcleanupskyfile.sql (revision 22161) @@ -0,0 +1,11 @@ +SELECT + diffSkyfile.*, + diffRun.state, + diffRun.workdir, + diffRun.dvodb, + diffRun.tess_id +FROM diffRun +JOIN diffSkyfile + USING(diff_id) +WHERE + diffRun.state = 'goto_cleaned' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_revertdiffskyfile_delete.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_revertdiffskyfile_delete.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_revertdiffskyfile_delete.sql (revision 22161) @@ -0,0 +1,5 @@ +DELETE FROM diffSkyfile +USING diffSkyfile, diffRun +WHERE + diffRun.diff_id = diffSkyfile.diff_id + AND fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_revertdiffskyfile_update.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_revertdiffskyfile_update.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_revertdiffskyfile_update.sql (revision 22161) @@ -0,0 +1,5 @@ +UPDATE diffRun +JOIN diffSkyfile USING(diff_id) +SET diffRun.state = 'new' +WHERE + diffSkyfile.fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_skyfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_skyfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_skyfile.sql (revision 22161) @@ -0,0 +1,29 @@ +SELECT + diffRun.skycell_id, + diffRun.tess_id, + diffRun.state, + diffSkyfile.*, + rawExp.exp_id, + rawExp.camera +FROM diffRun +JOIN diffSkyfile + USING(diff_id) +-- WS: my new stuff begins here +JOIN diffInputSkyfile + ON diffInputSkyfile.diff_id = diffRun.diff_id + AND diffInputSkyfile.template = 0 +JOIN warpRun + ON diffInputSkyfile.warp_id = warpRun.warp_id +JOIN fakeRun + ON warpRun.fake_id = fakeRun.fake_id +JOIN camRun + ON camRun.cam_id = fakeRun.cam_id +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id +-- WS: my new stuff ends here + WHERE + -- bogus test to allow appending conditionals + -- XXX EAM : this is not needed if we modify the difftool.c code : fix + diffRun.diff_id IS NOT NULL Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_todiffskyfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_todiffskyfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/difftool_todiffskyfile.sql (revision 22161) @@ -0,0 +1,78 @@ +-- This query is a little complicated, since it contains multiple self-joins. +-- This is because of the options involved: both templates and inputs can be either warps or stacks. +-- All of the templates and inputs need to exist, so we query against all of them. +SELECT DISTINCT + diffRun.diff_id, + diffRun.workdir, + diffRun.skycell_id, + diffRun.tess_id, + diffRun.label, + diffRun.state +FROM diffRun +-- Get list of templates for each diffRun +JOIN diffInputSkyfile AS diffTemplateSkyfile + ON diffRun.diff_id = diffTemplateSkyfile.diff_id + AND diffRun.skycell_id = diffTemplateSkyfile.skycell_id + AND diffTemplateSkyfile.template = 1 +-- Get list of inputs for each diffRun +JOIN diffInputSkyfile + ON diffRun.diff_id = diffInputSkyfile.diff_id + AND diffRun.skycell_id = diffInputSkyfile.skycell_id + AND diffInputSkyfile.template = 0 +-- Get warp templates +LEFT JOIN warpRun AS warpTemplateRun + ON warpTemplateRun.warp_id = diffTemplateSkyfile.warp_id + AND diffTemplateSkyfile.warp_id IS NOT NULL +LEFT JOIN warpSkyfile AS warpTemplateSkyfile + ON warpTemplateSkyfile.warp_id = warpTemplateRun.warp_id + AND warpTemplateSkyfile.skycell_id = diffTemplateSkyfile.skycell_id +-- Get warp inputs +LEFT JOIN warpRun + ON warpRun.warp_id = diffInputSkyfile.warp_id + AND diffInputSkyfile.warp_id IS NOT NULL +LEFT JOIN warpSkyfile + ON warpSkyfile.warp_id = warpRun.warp_id + AND warpSkyfile.skycell_id = diffInputSkyfile.skycell_id +-- Get stack templates +LEFT JOIN stackRun AS stackTemplateRun + ON stackTemplateRun.stack_id = diffTemplateSkyfile.stack_id +LEFT JOIN stackSumSkyfile AS stackTemplateSkyfile + ON stackTemplateSkyfile.stack_id = stackTemplateRun.stack_id +-- Get stack inputs +LEFT JOIN stackRun + ON stackRun.stack_id = diffInputSkyfile.stack_id + AND diffInputSkyfile.warp_id IS NULL +LEFT JOIN stackSumSkyfile + ON stackSumSkyfile.stack_id = stackRun.stack_id + AND diffInputSkyfile.warp_id IS NULL +-- Get what's already been processed +LEFT JOIN diffSkyfile + ON diffInputSkyfile.diff_id = diffSkyfile.diff_id +WHERE +-- Ready to be processed + ((diffRun.state = 'new' + AND diffSkyfile.diff_id IS NULL) + OR (diffRun.state = 'update' + AND diffSkyfile.data_state = 'cleaned') + ) +-- Ensure input warps are available + AND (diffInputSkyfile.warp_id IS NULL + OR (warpRun.state = 'full' + AND warpSkyfile.fault = 0 + AND warpSkyfile.ignored = 0)) +-- Ensure input stacks are available + AND (diffInputSkyfile.stack_id IS NULL + OR (stackRun.state = 'full' + AND stackSumSkyfile.fault = 0)) +-- Ensure template warps are available + AND (diffTemplateSkyfile.warp_id IS NULL + OR (warpTemplateRun.state = 'full' + AND warpTemplateSkyfile.fault = 0 + AND warpTemplateSkyfile.ignored = 0)) +-- Ensure template stacks are available + AND (diffTemplateSkyfile.stack_id IS NULL + OR (stackTemplateRun.state = 'full' + AND stackTemplateSkyfile.fault = 0)) + + + Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_completely_processed_exp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_completely_processed_exp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_completely_processed_exp.sql (revision 22161) @@ -0,0 +1,40 @@ +-- the output of this query must match the format of fakeRun row +SELECT DISTINCT + fake_id, + cam_id, + state, + workdir, + label, + reduction, + expgroup, + dvodb, + tess_id, + end_stage, + epoch +FROM + (SELECT + fakeRun.*, + rawImfile.class_id as rawimfile_class_id, + fakeProcessedImfile.class_id + FROM fakeRun + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + JOIN rawImfile + USING(exp_id) + LEFT JOIN fakeProcessedImfile + ON fakeRun.fake_id = fakeProcessedImfile.fake_id + AND rawImfile.exp_id = fakeProcessedImfile.exp_id + AND rawImfile.class_id = fakeProcessedImfile.class_id + WHERE + fakeRun.state = 'new' + GROUP BY + fakeRun.fake_id, + rawExp.exp_id + HAVING + COUNT(rawImfile.class_id) = COUNT(fakeProcessedImfile.class_id) + AND SUM(fakeProcessedImfile.fault) = 0 + ) as Foo Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_donecleanup.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_donecleanup.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_donecleanup.sql (revision 22161) @@ -0,0 +1,5 @@ +SELECT + fakeRun.* +FROM fakeRun +WHERE + fakeRun.state = 'cleaned' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_find_camrun.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_find_camrun.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_find_camrun.sql (revision 22161) @@ -0,0 +1,37 @@ +SELECT + * +FROM + (SELECT DISTINCT + camRun.cam_id, + chipRun.*, + rawExp.camera, + rawExp.telescope, + rawExp.dateobs, + rawExp.exp_tag, + rawExp.exp_type, + rawExp.filelevel, + rawExp.filter, + rawExp.airmass, + rawExp.ra, + rawExp.decl, + rawExp.exp_time, + rawExp.sat_pixel_frac, + rawExp.bg, + rawExp.bg_stdev, + rawExp.bg_mean_stdev, + rawExp.alt, + rawExp.az, + rawExp.ccd_temp, + rawExp.posang, + rawExp.object, + rawExp.solang, + rawExp.comment + FROM camRun + JOIN chipRun + using(chip_id) + JOIN rawExp + using(exp_id) + WHERE + camRun.state = 'full' + AND chipRun.state = 'full' +) as Foo Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_find_pendingexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_find_pendingexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_find_pendingexp.sql (revision 22161) @@ -0,0 +1,34 @@ +-- this query is used by both camtool -pendingexp & camtool -addprocessedexp it +-- does a little more work then is necessary for -addprocessed but it seems +-- "cleaner" to use the same query for both cases +SELECT * FROM + (SELECT + fakeRun.*, + rawExp.exp_tag, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel + FROM fakeRun + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN chipProcessedImfile + USING(chip_id) + JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id + LEFT JOIN fakeProcessedImfile + USING(fake_id) + LEFT JOIN fakeMask + ON fakeRun.label = fakeMask.label + WHERE + fakeRun.state = 'new' + AND camRun.state = 'full' + AND chipRun.state = 'full' + AND fakeMask.label IS NULL + AND fakeProcessedImfile.fake_id IS NOT NULL + GROUP BY + fakeRun.fake_id + ) as Foo Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_pendingcleanupimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_pendingcleanupimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_pendingcleanupimfile.sql (revision 22161) @@ -0,0 +1,15 @@ +SELECT + fakeProcessedImfile.*, + fakeRun.state, + fakeRun.workdir, + fakeRun.label, + fakeRun.reduction, + fakeRun.expgroup, + fakeRun.dvodb, + fakeRun.tess_id, + fakeRun.end_stage +FROM fakeRun +JOIN fakeProcessedImfile + USING(fake_id) +WHERE + fakeRun.state = 'goto_cleaned' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_pendingcleanuprun.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_pendingcleanuprun.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_pendingcleanuprun.sql (revision 22161) @@ -0,0 +1,13 @@ +SELECT + fakeRun.fake_id, + rawExp.camera, + fakeRun.state +FROM fakeRun +JOIN camRun +USING (cam_id) +JOIN chipRun +USING (chip_id) +JOIN rawExp +USING (exp_id) +WHERE + fakeRun.state = 'goto_cleaned' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_pendingimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_pendingimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_pendingimfile.sql (revision 22161) @@ -0,0 +1,51 @@ +SELECT DISTINCT * FROM ( + -- the subselect is so where criteria can be specified without knowing + -- which table the field came from + SELECT + fakeRun.*, + chipProcessedImfile.exp_id, + chipProcessedImfile.class_id, + chipProcessedImfile.uri, + chipProcessedImfile.bg, + chipProcessedImfile.bg_stdev, + chipProcessedImfile.bg_mean_stdev, + chipProcessedImfile.fringe_0, + chipProcessedImfile.fringe_1, + chipProcessedImfile.fringe_2, + chipProcessedImfile.ap_resid, + chipProcessedImfile.ap_resid_stdev, + chipProcessedImfile.sigma_ra, + chipProcessedImfile.sigma_dec, + chipProcessedImfile.n_stars, + chipProcessedImfile.n_extended, + chipProcessedImfile.n_cr, + chipProcessedImfile.n_astrom, + chipProcessedImfile.path_base as chip_path_base, + camProcessedExp.path_base as cam_path_base, + rawExp.exp_name, + rawExp.exp_tag, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel + FROM fakeRun + JOIN camRun USING(cam_id) + JOIN camProcessedExp USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN chipProcessedImfile USING(chip_id) + JOIN rawExp + ON chipProcessedImfile.exp_id = rawExp.exp_id + LEFT JOIN fakeProcessedImfile + ON fakeRun.fake_id = fakeProcessedImfile.fake_id + AND chipProcessedImfile.class_id = fakeProcessedImfile.class_id + LEFT JOIN fakeMask + ON fakeRun.label = fakeMask.label + WHERE + ((fakeRun.state = 'new' + AND fakeProcessedImfile.fake_id IS NULL + AND fakeProcessedImfile.class_id IS NULL + ) + OR (fakeRun.state = 'update' + AND fakeProcessedImfile.data_state = 'cleaned') + ) + AND fakeMask.label IS NULL + ) as fakePendingImfile Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_processedimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_processedimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_processedimfile.sql (revision 22161) @@ -0,0 +1,17 @@ +SELECT DISTINCT + fakeProcessedImfile.*, + rawExp.exp_tag, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel +FROM fakeRun +JOIN fakeProcessedImfile + USING(fake_id) +JOIN rawExp + ON fakeProcessedImfile.exp_id = rawExp.exp_id +WHERE +-- bogus test; just here so there there is a 'WHERE' stmt to append +-- conditionals too + fakeProcessedImfile.exp_id is NOT NULL + Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_queue_cam_id.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_queue_cam_id.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_queue_cam_id.sql (revision 22161) @@ -0,0 +1,17 @@ +INSERT INTO fakeRun + SElECT + 0, -- fake_id + cam_id, -- cam_id + '%s', -- state + '%s', -- workdir + '%s', -- label + '%s', -- reduction + '%s', -- expgroup + '%s', -- dvodb + '%s', -- tess_id + '%s', -- end_stage + NULL -- epoch + FROM camRun + WHERE + camRun.state = 'full' + AND camRun.cam_id = %lld Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_revertprocessedimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_revertprocessedimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_revertprocessedimfile.sql (revision 22161) @@ -0,0 +1,8 @@ +DELETE FROM fakeProcessedImfile +USING fakeProcessedImfile, fakeRun, camRun, chipRun, rawExp +WHERE + fakeProcessedImfile.fake_id = fakeRun.fake_id + AND fakeRun.cam_id = camRun.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND fakeProcessedImfile.fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_unmasked.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_unmasked.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/faketool_unmasked.sql (revision 22161) @@ -0,0 +1,11 @@ +SELECT + * +FROM + (SELECT + fakeRun.label, + count(fakeRun.fake_id) as n_fakeruns + FROM fakeRun + LEFT JOIN fakeMask + USING (label) + WHERE fakeMask.label IS NULL + GROUP BY fakeRun.label) as fakeUnmask Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_chiprundone.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_chiprundone.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_chiprundone.sql (revision 22161) @@ -0,0 +1,15 @@ +SELECT + corr_id, + cam_id, + chipRun.* +FROM flatcorrRun +JOIN flatcorrChipLink + USING(corr_id) +JOIN chipRun + USING(chip_id) +LEFT JOIN camRun + ON chipRun.chip_id = camRun.chip_id +WHERE + flatcorrRun.state = 'new' + AND chipRun.state = 'full' + AND camRun.cam_id is NULL; Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_completely_processed_chiprun.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_completely_processed_chiprun.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_completely_processed_chiprun.sql (revision 22161) @@ -0,0 +1,36 @@ +INSERT INTO chipRunDone +SELECT + chip_id, + exp_id +FROM (SELECT DISTINCT + chip_id, + exp_id, + state, + workdir, + workdir_state, + label, + reduction, + expgroup, + dvodb + FROM + (SELECT + chipRun.*, + rawImfile.class_id as rawimfile_class_id, + chipProcessedImfile.class_id + FROM chipRun + JOIN rawImfile + USING(exp_id) + LEFT JOIN chipProcessedImfile + ON chipRun.chip_id = chipProcessedImfile.chip_id + AND rawImfile.exp_id = chipProcessedImfile.exp_id + AND rawImfile.class_id = chipProcessedImfile.class_id + WHERE + chipRun.state = 'full' + GROUP BY + chipRun.chip_id, + chipRun.exp_id + HAVING + COUNT(rawImfile.class_id) = COUNT(chipProcessedImfile.class_id) + AND SUM(chipProcessedImfile.fault) = 0 + ) as Foo + ) as Bar Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_create_tmp_chiprundone.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_create_tmp_chiprundone.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_create_tmp_chiprundone.sql (revision 22161) @@ -0,0 +1,3 @@ +CREATE TEMPORARY TABLE chipRunDone +(chip_id BIGINT, exp_id VARCHAR(64), PRIMARY KEY(chip_id)) ENGINE=MEMORY + Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_find_completed_floatcorruns.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_find_completed_floatcorruns.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_find_completed_floatcorruns.sql (revision 22161) @@ -0,0 +1,13 @@ +SELECT DISTINCT + flatcorrRun.* +FROM flatcorrRun +JOIN flatcorrExp + USING(corr_id) +JOIN chipRunDone + USING(chip_id) +WHERE + flatcorrRun.state != "stop" +GROUP BY + corr_id, chip_id +HAVING + COUNT(flatcorrExp.chip_id) = COUNT(chipRunDone.chip_id) Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_find_processedimfiles.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_find_processedimfiles.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_find_processedimfiles.sql (revision 22161) @@ -0,0 +1,14 @@ +SELECT + corr_id, + chipProcessedImfile.* +FROM flatcorrRun +JOIN flatcorrExp + USING(corr_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + USING(chip_id) +WHERE + flatcorrRun.state = 'run' + AND chipRun.state = 'new' + Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_inputimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_inputimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_inputimfile.sql (revision 22161) @@ -0,0 +1,9 @@ +SELECT + chipProcessedImfile.*, + rawExp.filelevel , + rawExp.camera, + rawExp.telescope, + rawExp.filter +FROM chipProcessedImfile +JOIN rawExp +USING (exp_id) Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_pending.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_pending.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_pending.sql (revision 22161) @@ -0,0 +1,24 @@ +SELECT + * +FROM +flatcorrRun +JOIN flatcorrExp + USING(corr_id) +JOIN chipRun + USING(chip_id) +JOIN rawImfile + USING(exp_id) +LEFT JOIN chipProcessedImfile + ON chipRun.chip_id = chipProcessedImfile.chip_id + AND rawImfile.exp_id = chipProcessedImfile.exp_id + AND rawImfile.class_id = chipProcessedImfile.class_id +WHERE + flatcorrRun.state = 'run' + AND chipRun.state = 'full' +GROUP BY + chipRun.chip_id, + chipRun.exp_id +HAVING + COUNT(rawImfile.class_id) = COUNT(chipProcessedImfile.class_id) + AND SUM(chipProcessedImfile.fault) = 0 + Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_pendingprocess.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_pendingprocess.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/flatcorr_pendingprocess.sql (revision 22161) @@ -0,0 +1,34 @@ +SELECT + corr_id, + dvodb, + filter, + state, + workdir, + label, + reduction, + region, + chip_count, + cam_count +FROM + (SELECT + flatcorrRun.*, + flatcorrChipLink.corr_id as chip_corr_id, + count(flatcorrChipLink.chip_id) as chip_count + FROM flatcorrRun + JOIN flatcorrChipLink + ON flatcorrChipLink.corr_id = flatcorrRun.corr_id + GROUP BY + flatcorrChipLink.corr_id) AS t1 +LEFT JOIN + (SELECT + flatcorrCamLink.corr_id as cam_corr_id, + count(flatcorrCamLink.cam_id) as cam_count + FROM flatcorrCamLink + JOIN camRun + ON flatcorrCamLink.cam_id = camRun.cam_id + WHERE camRun.state = 'full' + GROUP BY + flatcorrCamLink.corr_id) AS t2 +ON t1.chip_corr_id = t2.cam_corr_id +WHERE chip_count = cam_count +AND state = 'new' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_addmask.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_addmask.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_addmask.sql (revision 22161) @@ -0,0 +1,6 @@ +UPDATE + magicRun +SET + state = 'stop' +WHERE + state != 'stop' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_create_tmp_warpcomplete.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_create_tmp_warpcomplete.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_create_tmp_warpcomplete.sql (revision 22161) @@ -0,0 +1,3 @@ +CREATE TEMPORARY TABLE warpComplete + (warp_id BIGINT, skycell_id VARCHAR(64), tess_id VARCHAR(64), + PRIMARY KEY(warp_id, skycell_id, tess_id)) ENGINE=MEMORY; Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery.sql (revision 22161) @@ -0,0 +1,110 @@ +-- This is a sequence of SQL for searching for exposures on which +-- magic may be run, and populating the run. This file is provided as +-- a convenience, and is intended solely for playing around and +-- testing. Updates here will not affect the operation of magictool. +-- Conversely, it may not be up to date with the actual queries used +-- by magictool. + +-------------------------------------------------------------------- +-- NOTE: THIS FILE IS NOT USED BY magictool +-- NOTE : thi sfile contains examples of tess_id as a join qualifier which are wrong +-------------------------------------------------------------------- + +-- magictool_definebyquery_temp_create.sql +CREATE TEMPORARY TABLE magicBestDiffs ( +exp_id BIGINT, +skycell_id VARCHAR(64), +tess_id VARCHAR(64), +diff_id BIGINT, +PRIMARY KEY(exp_id, skycell_id, tess_id) +) ENGINE=MEMORY; + + +-- magictool_definebyquery_temp_insert.sql +-- List of best differences for each exposure +INSERT INTO magicBestDiffs +SELECT + exp_id, + warpSkyfile.skycell_id, + warpSkyfile.tess_id, + MAX(diffSkyfile.diff_id) AS diff_id +FROM rawExp +JOIN chipRun USING(exp_id, tess_id) +JOIN camRun USING(chip_id, tess_id) +JOIN fakeRun USING(cam_id, tess_id) +JOIN warpRun USING(fake_id, tess_id) +JOIN warpSkyCellMap USING(warp_id, tess_id) +JOIN warpSkyfile USING(warp_id, skycell_id, tess_id) +JOIN diffInputSkyfile + ON diffInputSkyfile.warp_id = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.tess_id = warpSkyfile.tess_id + AND diffInputSkyfile.template = 0 -- selecting inputs only +JOIN diffSkyfile USING(diff_id) +WHERE + diffSkyfile.fault = 0 +----- AND warpSkyfile.good_frac >= 0.5 +-- magictool_definebyquery_temp_insert_groupby.sql +GROUP BY + exp_id, + warpSkyfile.skycell_id +; + + +-- magictool_definebyquery_select_part1.sql +-- This is part 1 of 2 of a query to get a list of exposures on which magic may be performed +-- After this follows magictool_definebyquery_select_part2.sql +SELECT + exp_id, + filter, + num_todo, + num_done +FROM ( + -- Number of skycells as a function of exposure + SELECT + exp_id, + filter, + COUNT(skycell_id) AS num_todo + FROM rawExp + JOIN chipRun USING(exp_id, tess_id) + JOIN camRun USING(chip_id, tess_id) + JOIN fakeRun USING(cam_id, tess_id) + JOIN warpRun USING(fake_id, tess_id) + JOIN warpSkyCellMap USING(warp_id, tess_id) + JOIN warpSkyfile USING(warp_id, skycell_id, tess_id) + WHERE + warpSkyfile.ignored = 0 + AND warpRun.state = 'stop' + -- INSERT HERE any additional restrictions (e.g., exp_id, warpSkyfile.good_frac) +----- AND warpSkyfile.good_frac > 0.5 +-- INSERT HERE magictool_definebyquery_select_part2.sql +-- magictool_definebyquery_select_part2.sql +-- This is part 2 of 2 of a query to get a list of exposures on which magic may be performed +-- This follows magictool_definebyquery_select_part1.sql + GROUP BY + exp_id + ) AS magicSkycellNums +JOIN ( + -- Number of completed diffs for an exposure + SELECT + exp_id, + COUNT(diff_id) AS num_done + FROM magicBestDiffs + GROUP BY + exp_id + ) AS magicDiffNums USING(exp_id) +LEFT JOIN magicRun USING(exp_id) +WHERE + num_done = num_todo + AND magicRun.magic_id IS NULL +; + +-- magictool_definebyquery_insert.sql +-- Insert the best list of diffs as magic inputs +INSERT INTO magicInputSkyfile +SELECT + @MAGIC_ID@, -- Update this with the appropriate magic_id + diff_id, + CONCAT(tess_id, '.', skycell_id) AS node +FROM magicBestDiffs +; Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_insert.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_insert.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_insert.sql (revision 22161) @@ -0,0 +1,9 @@ +-- Insert the best list of diffs as magic inputs +INSERT INTO magicInputSkyfile +SELECT + @MAGIC_ID@, -- Update this with the appropriate magic_id + diff_id, + skycell_id +FROM magicBestDiffs +WHERE + exp_id = @EXP_ID@ -- Update this with the appropriate exp_id Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_select_part1.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_select_part1.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_select_part1.sql (revision 22161) @@ -0,0 +1,25 @@ +-- This is part 1 of 2 of a query to get a list of exposures on which magic may be performed +-- After this follows magictool_definebyquery_select_part2.sql +SELECT + exp_id, + filter, + num_todo, + num_done +FROM ( + -- Number of skycells as a function of exposure + SELECT + exp_id, + filter, + COUNT(skycell_id) AS num_todo + FROM rawExp + JOIN chipRun USING(exp_id) + JOIN camRun USING(chip_id) + JOIN fakeRun USING(cam_id) + JOIN warpRun USING(fake_id) + JOIN warpSkyCellMap USING(warp_id) + JOIN warpSkyfile USING(warp_id, skycell_id) + WHERE + warpSkyfile.ignored = 0 + AND warpRun.state = 'stop' + -- INSERT HERE any additional restrictions (e.g., exp_id, warpSkyfile.good_frac) +-- INSERT HERE magictool_definebyquery_select_part2.sql Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_select_part2.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_select_part2.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_select_part2.sql (revision 22161) @@ -0,0 +1,18 @@ +-- This is part 2 of 2 of a query to get a list of exposures on which magic may be performed +-- This follows magictool_definebyquery_select_part1.sql + GROUP BY + exp_id + ) AS magicSkycellNums +JOIN ( + -- Number of completed diffs for an exposure + SELECT + exp_id, + COUNT(diff_id) AS num_done + FROM magicBestDiffs + GROUP BY + exp_id + ) AS magicDiffNums USING(exp_id) +LEFT JOIN magicRun USING(exp_id) +WHERE + num_done = num_todo + AND magicRun.magic_id IS NULL Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_select_test.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_select_test.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_select_test.sql (revision 22161) @@ -0,0 +1,46 @@ +-- This is the combination of the two parts of the query to get a list +-- of exposures to be magic-ed + +-------------------------------------------------------------------------------- +-- THIS FILE IS INTENDED FOR TESTING ONLY! IT IS NOT USED BY stacktool. +-- CHANGES SHOULD BE MADE TO magictool_definebyquery_select_part1.sql +-- AND magictool_definebyquery_select_part2.sql +-------------------------------------------------------------------------------- + +-- magictool_definebyquery_select_part1.sql +SELECT + * +FROM ( + -- Number of skycells as a function of exposure + SELECT + exp_id, + filter, + COUNT(skycell_id) AS num_todo + FROM rawExp + JOIN chipRun USING(exp_id) + JOIN camRun USING(chip_id) + JOIN fakeRun USING(cam_id) + JOIN warpRun USING(fake_id) + JOIN warpSkyCellMap USING(warp_id) + JOIN warpSkyfile USING(warp_id, skycell_id) + WHERE + warpSkyfile.ignored = 0 + AND warpRun.state = 'stop' + -- INSERT HERE any additional restrictions (e.g., exp_id, warpSkyfile.good_frac) +-- magictool_definebyquery_select_part2.sql + GROUP BY + exp_id + ) AS magicSkycellNums +JOIN ( + -- Number of completed diffs for an exposure + SELECT + exp_id, + COUNT(diff_id) AS num_done + FROM magicBestDiffs + GROUP BY + exp_id + ) AS magicDiffNums USING(exp_id) +LEFT JOIN magicRun USING(exp_id) +WHERE + num_done = num_todo + AND magicRun.magic_id IS NULL Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_temp_create.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_temp_create.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_temp_create.sql (revision 22161) @@ -0,0 +1,7 @@ +CREATE TEMPORARY TABLE magicBestDiffs ( +exp_id BIGINT, +skycell_id VARCHAR(64), +tess_id VARCHAR(64), +diff_id BIGINT, +PRIMARY KEY(exp_id, skycell_id, tess_id) +) ENGINE=MEMORY; Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_temp_insert.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_temp_insert.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_temp_insert.sql (revision 22161) @@ -0,0 +1,21 @@ +-- List of best differences for each exposure +INSERT INTO magicBestDiffs +SELECT + rawExp.exp_id, + warpSkyfile.skycell_id, + warpSkyfile.tess_id, + MAX(diffSkyfile.diff_id) AS diff_id +FROM rawExp +JOIN chipRun USING(exp_id) +JOIN camRun USING(chip_id) +JOIN fakeRun USING(cam_id) +JOIN warpRun USING(fake_id) +JOIN warpSkyCellMap USING(warp_id) +JOIN warpSkyfile USING(warp_id, skycell_id) +JOIN diffInputSkyfile + ON diffInputSkyfile.warp_id = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.template = 0 -- selecting inputs only +JOIN diffSkyfile USING(diff_id) +WHERE + diffSkyfile.fault = 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_temp_insert_groupby.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_temp_insert_groupby.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_definebyquery_temp_insert_groupby.sql (revision 22161) @@ -0,0 +1,3 @@ +GROUP BY + exp_id, + warpSkyfile.skycell_id Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_inputs.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_inputs.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_inputs.sql (revision 22161) @@ -0,0 +1,33 @@ +SELECT * +FROM ( +-- Single skycells +SELECT + magicRun.magic_id, + magicRun.state, + magicInputSkyfile.node, + diffSkyfile.uri, + diffSkyfile.path_base, + diffSkyfile.fault +FROM magicInputSkyfile +JOIN diffSkyfile + USING(diff_id) +JOIN magicRun + USING(magic_id) +UNION +-- Merged skycells +SELECT + magicRun.magic_id, + magicRun.state, + magicTree.node, + magicNodeResult.uri, + NULL, -- magicNodeResult doesn't have a path_base + magicNodeResult.fault +FROM magicTree +JOIN magicRun + USING(magic_id) +JOIN magicNodeResult + ON magicTree.magic_id = magicNodeResult.magic_id + AND magicTree.dep = magicNodeResult.node +) as Foo +WHERE + fault = 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_inputskyfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_inputskyfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_inputskyfile.sql (revision 22161) @@ -0,0 +1,7 @@ +SELECT + magicInputSkyfile.* +FROM magicInputSkyfile +JOIN magicRun + USING(magic_id) +WHERE + magicRun.state = 'run' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_mask.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_mask.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_mask.sql (revision 22161) @@ -0,0 +1,8 @@ +SELECT + magicMask.* +FROM magicMask +JOIN magicRun + USING(magic_id) +WHERE + magicRun.state = 'stop' + AND magicMask.fault = 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_tomask.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_tomask.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_tomask.sql (revision 22161) @@ -0,0 +1,18 @@ +SELECT + magic_id, + exp_id, + camera, + magicRun.workdir, + uri +FROM magicRun +JOIN rawExp USING(exp_id) +JOIN magicTree USING(magic_id) +LEFT JOIN magicNodeResult USING(magic_id, node) +WHERE + magicRun.state = 'run' + AND magicNodeResult.node = 'root' + AND magicNodeResult.fault = 0 +GROUP BY + magic_id +HAVING + COUNT(magicTree.node) = COUNT(magicNodeResult.node) Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_toprocess_inputs.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_toprocess_inputs.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_toprocess_inputs.sql (revision 22161) @@ -0,0 +1,19 @@ +SELECT + magicTree.*, + rawExp.camera, + diffSkyfile.path_base, + -- convert magic_id into a boolean value (1 or 0) + -- note that the type stays a 64 bit int + magicNodeResult.magic_id IS TRUE as done +FROM magicTree +JOIN magicRun USING(magic_id) +JOIN magicInputSkyfile USING(magic_id, node) +JOIN diffSkyfile USING(diff_id) +JOIN rawExp USING(exp_id) +LEFT JOIN magicNodeResult + ON magicTree.magic_id = magicNodeResult.magic_id + AND magicTree.node = magicNodeResult.node +WHERE + magicRun.state = 'run' + AND magicNodeResult.magic_id IS NULL + AND magicNodeResult.node IS NULL Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_toprocess_tree.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_toprocess_tree.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_toprocess_tree.sql (revision 22161) @@ -0,0 +1,17 @@ +SELECT + magicTree.*, + exp_id, + camera, + magicRun.workdir, + -- convert magic_id into a boolean value (1 or 0) + -- note that the type stays a 64 bit int + magicNodeResult.magic_id IS TRUE as done, + magicNodeResult.fault IS TRUE as bad +FROM magicTree +JOIN magicRun USING(magic_id) +JOIN rawExp USING(exp_id) +LEFT JOIN magicNodeResult + ON magicTree.magic_id = magicNodeResult.magic_id + AND magicTree.node = magicNodeResult.node +WHERE + magicRun.state = 'run' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_toskyfilemask.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_toskyfilemask.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_toskyfilemask.sql (revision 22161) @@ -0,0 +1,10 @@ +SELECT + magicMask.* +FROM magicRun +JOIN magicMask + USING(magic_id) +LEFT JOIN magicSkyfileMask + USING(magic_id) +WHERE + magicRun.state = 'run' + AND magicSkyfileMask.magic_id IS NULL Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_totree.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_totree.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/magictool_totree.sql (revision 22161) @@ -0,0 +1,16 @@ +SELECT + magic_id, + exp_id, + camera, + workdir, + ra, + decl, + tess_id +FROM magicRun +JOIN rawExp USING(exp_id) +LEFT JOIN magicTree + USING(magic_id) +WHERE + magicRun.state = 'run' + AND magicTree.node IS NULL + AND magicRun.fault = 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/pstamptool_addjob_otherjob.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/pstamptool_addjob_otherjob.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/pstamptool_addjob_otherjob.sql (revision 22161) @@ -0,0 +1,3 @@ +INSERT INTO pstampJob + (req_id, rownum, state, jobType, uri, exp_id, outputBase, fault) + VALUES( %lld, '%s', '%s', '%s', '%s', %lld, '%s', %s) Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/pstamptool_addjob_stampjob.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/pstamptool_addjob_stampjob.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/pstamptool_addjob_stampjob.sql (revision 22161) @@ -0,0 +1,3 @@ +INSERT INTO pstampJob + (req_id, rownum, state, jobType, uri, exp_id, outputBase, fault, args) + VALUES( %lld, '%s', '%s', '%s', '%s', %lld, '%s', %s, '%s') Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/pstamptool_datastore.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/pstamptool_datastore.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/pstamptool_datastore.sql (revision 22161) @@ -0,0 +1,1 @@ +SELECT * FROM pstampDataStore Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/pstamptool_pendingjob.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/pstamptool_pendingjob.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/pstamptool_pendingjob.sql (revision 22161) @@ -0,0 +1,4 @@ +SELECT + * + FROM pstampJob + WHERE state = 'run' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/pstamptool_pendingreq.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/pstamptool_pendingreq.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/pstamptool_pendingreq.sql (revision 22161) @@ -0,0 +1,4 @@ +SELECT + * + FROM pstampRequest + WHERE state = 'new' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/pxadmin_create_tables.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/pxadmin_create_tables.sql (revision 22161) @@ -0,0 +1,1050 @@ +CREATE TABLE pzDataStore ( + camera VARCHAR(64), + telescope VARCHAR(64), + uri VARCHAR(255), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(camera, telescope) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE summitExp ( + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + dateobs DATETIME, + exp_type VARCHAR(64), + uri VARCHAR(255), + imfiles INT, + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_name, camera, telescope), + KEY(fault)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE summitImfile ( + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + file_id VARCHAR(64), + bytes INT, + md5sum VARCHAR(32), + class VARCHAR(64), + class_id VARCHAR(64), + uri VARCHAR(255), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_name, camera, telescope, class, class_id), + KEY(file_id), + FOREIGN KEY (exp_name, camera, telescope) + REFERENCES summitExp(exp_name, camera, telescope)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pzDownloadExp ( + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + state VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_name, camera, telescope), + KEY(state), + FOREIGN KEY (exp_name, camera, telescope) + REFERENCES summitExp(exp_name, camera, telescope)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pzDownloadImfile ( + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + class VARCHAR(64), + class_id VARCHAR(64), + uri VARCHAR(255), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_name, camera, telescope, class, class_id), + KEY(fault), + FOREIGN KEY (exp_name, camera, telescope) + REFERENCES pzDownloadExp(exp_name, camera, telescope), + FOREIGN KEY (exp_name, camera, telescope, class, class_id) + REFERENCES summitImfile(exp_name, camera, telescope, class, class_id)) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE newExp ( + exp_id BIGINT AUTO_INCREMENT, + tmp_exp_name VARCHAR(64), + tmp_camera VARCHAR(64), + tmp_telescope VARCHAR(64), + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + reduction VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + label VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_id), + KEY(exp_id), + KEY(tmp_exp_name), + KEY(tmp_camera), + KEY(tmp_telescope), + KEY(state), + KEY(workdir_state), + KEY(end_stage), + KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE newImfile ( + exp_id BIGINT, + tmp_class_id VARCHAR(64), + uri VARCHAR(255), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_id, tmp_class_id), + FOREIGN KEY (exp_id) + REFERENCES newExp(exp_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE rawExp ( + exp_id BIGINT, + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + dateobs DATETIME, + exp_tag VARCHAR(255), + exp_type VARCHAR(64), + filelevel VARCHAR(64), + workdir VARCHAR(255), + reduction VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + filter VARCHAR(64), + comment VARCHAR(80), + airmass FLOAT, + ra DOUBLE, + decl DOUBLE, + exp_time FLOAT, + sat_pixel_frac FLOAT, + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + alt DOUBLE, + az DOUBLE, + ccd_temp FLOAT, + posang DOUBLE, + m1_x FLOAT, + m1_y FLOAT, + m1_z FLOAT, + m1_tip FLOAT, + m1_tilt FLOAT, + m2_x FLOAT, + m2_y FLOAT, + m2_z FLOAT, + m2_tip FLOAT, + m2_tilt FLOAT, + env_temperature FLOAT, + env_humidity FLOAT, + env_wind_speed FLOAT, + env_wind_dir FLOAT, + teltemp_m1 FLOAT, + teltemp_m1cell FLOAT, + teltemp_m2 FLOAT, + teltemp_spider FLOAT, + teltemp_truss FLOAT, + teltemp_extra FLOAT, + pon_time FLOAT, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + object VARCHAR(64), + solang FLOAT, + hostname VARCHAR(64), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_id), + KEY(exp_name), + KEY(end_stage), + KEY(fault), + FOREIGN KEY (exp_id) + REFERENCES newExp(exp_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE rawImfile ( + exp_id BIGINT, + exp_name VARCHAR(64), + camera VARCHAR(64), + telescope VARCHAR(64), + dateobs DATETIME, + tmp_class_id VARCHAR(64), + class_id VARCHAR(64), + uri VARCHAR(255), + exp_type VARCHAR(64), + filelevel VARCHAR(64), + filter VARCHAR(64), + comment VARCHAR(80), + airmass FLOAT, + ra DOUBLE, + decl DOUBLE, + exp_time FLOAT, + sat_pixel_frac FLOAT, + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + alt DOUBLE, + az DOUBLE, + ccd_temp FLOAT, + posang DOUBLE, + m1_x FLOAT, + m1_y FLOAT, + m1_z FLOAT, + m1_tip FLOAT, + m1_tilt FLOAT, + m2_x FLOAT, + m2_y FLOAT, + m2_z FLOAT, + m2_tip FLOAT, + m2_tilt FLOAT, + env_temperature FLOAT, + env_humidity FLOAT, + env_wind_speed FLOAT, + env_wind_dir FLOAT, + teltemp_m1 FLOAT, + teltemp_m1cell FLOAT, + teltemp_m2 FLOAT, + teltemp_spider FLOAT, + teltemp_truss FLOAT, + teltemp_extra FLOAT, + pon_time FLOAT, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + object VARCHAR(64), + hostname VARCHAR(64), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(exp_id, class_id), + KEY(tmp_class_id), + KEY(fault), + UNIQUE KEY(exp_id, tmp_class_id), + FOREIGN KEY (exp_id, tmp_class_id) + REFERENCES newImfile(exp_id, tmp_class_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE guidePendingExp (guide_id BIGINT AUTO_INCREMENT, exp_id BIGINT, recipe VARCHAR(64), PRIMARY KEY(guide_id), KEY(guide_id), KEY(exp_id)) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE chipRun ( + chip_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + reduction VARCHAR(64), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + PRIMARY KEY(chip_id), + KEY(chip_id), KEY(exp_id), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(expgroup), + KEY(end_stage), + INDEX(chip_id, exp_id), + FOREIGN KEY (exp_id) + REFERENCES rawExp(exp_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE chipProcessedImfile ( + chip_id BIGINT, + exp_id BIGINT, + class_id VARCHAR(64), + data_state VARCHAR(64), + uri VARCHAR(255), + bg FLOAT, + bg_stdev FLOAT, + bg_mean_stdev FLOAT, + bias FLOAT, + bias_stdev FLOAT, + fringe_0 FLOAT, + fringe_1 FLOAT, + fringe_2 FLOAT, + sigma_ra FLOAT, + sigma_dec FLOAT, + ap_resid FLOAT, + ap_resid_stdev FLOAT, + zp_mean FLOAT, + zp_stdev FLOAT, + fwhm_major FLOAT, + fwhm_minor FLOAT, + dtime_detrend FLOAT, + dtime_photom FLOAT, + dtime_astrom FLOAT, + hostname VARCHAR(64), + n_stars INT, + n_extended INT, + n_cr INT, + n_astrom INT, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(chip_id, exp_id, class_id), + KEY(data_state), + KEY(fault), + FOREIGN KEY (chip_id, exp_id) + REFERENCES chipRun(chip_id, exp_id), + FOREIGN KEY (exp_id, class_id) + REFERENCES rawImfile(exp_id, class_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE chipMask ( + label VARCHAR(64), + PRIMARY KEY(label)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE camRun ( + cam_id BIGINT AUTO_INCREMENT, + chip_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + reduction VARCHAR(64), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + PRIMARY KEY(cam_id), + KEY(cam_id), + KEY(chip_id), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(expgroup), + KEY(end_stage), + INDEX(cam_id, chip_id), + FOREIGN KEY (chip_id) + REFERENCES chipRun(chip_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE camProcessedExp ( + cam_id BIGINT, + uri VARCHAR(255), + bg FLOAT, + bg_stdev FLOAT, + bg_mean_stdev FLOAT, + bias FLOAT, + bias_stdev FLOAT, + fringe_0 FLOAT, + fringe_1 FLOAT, + fringe_2 FLOAT, + sigma_ra FLOAT, + sigma_dec FLOAT, + ap_resid FLOAT, + ap_resid_stdev FLOAT, + zp_mean FLOAT, + zp_stdev FLOAT, + fwhm_major FLOAT, + fwhm_minor FLOAT, + dtime_detrend FLOAT, + dtime_photom FLOAT, + dtime_astrom FLOAT, + hostname VARCHAR(64), + n_stars INT, + n_extended INT, + n_cr INT, + n_astrom INT, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(cam_id), + KEY(fault), + FOREIGN KEY (cam_id) + REFERENCES camRun(cam_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE camMask ( + label VARCHAR(64), + PRIMARY KEY(label)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fakeRun ( + fake_id BIGINT AUTO_INCREMENT, + cam_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + reduction VARCHAR(64), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(fake_id), + KEY(cam_id), + KEY(state), + KEY(label), + KEY(expgroup), + KEY(end_stage), + INDEX(fake_id, cam_id), + FOREIGN KEY (cam_id) + REFERENCES camRun(cam_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fakeProcessedImfile ( + fake_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + dtime_fake FLOAT, + hostname VARCHAR(64), + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(fake_id, exp_id, class_id), + KEY(fault), + FOREIGN KEY (fake_id) + REFERENCES fakeRun(fake_id), + FOREIGN KEY (exp_id, class_id) + REFERENCES rawImfile(exp_id, class_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fakeMask ( + label VARCHAR(64), + PRIMARY KEY(label)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detRun ( + det_id BIGINT AUTO_INCREMENT, + iteration INT, + det_type VARCHAR(64), + mode VARCHAR(64), + state VARCHAR(64), + filelevel VARCHAR(64), + workdir VARCHAR(255), + camera VARCHAR(64), + telescope VARCHAR(64), + exp_type VARCHAR(64), + reduction VARCHAR(64), + filter VARCHAR(64), + airmass_min FLOAT, + airmass_max FLOAT, + exp_time_min FLOAT, + exp_time_max FLOAT, + ccd_temp_min FLOAT, + ccd_temp_max FLOAT, + posang_min DOUBLE, + posang_max DOUBLE, + registered DATETIME, + time_begin DATETIME, + time_end DATETIME, + use_begin DATETIME, + use_end DATETIME, + solang_min FLOAT, + solang_max FLOAT, + label VARCHAR(64), + ref_det_id BIGINT, + ref_iter INT, + -- parent INT, :: dropping this + PRIMARY KEY(det_id), + KEY(det_id), + KEY(iteration), + KEY(det_type), + KEY(mode), + KEY(state), + KEY(label), + -- KEY(parent), :: dropping this + INDEX(det_id, iteration)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detInputExp ( +det_id BIGINT, +iteration INT, +exp_id BIGINT, +include TINYINT, +PRIMARY KEY(det_id, iteration, exp_id), +INDEX(det_id, exp_id), +INDEX(det_id, iteration), +FOREIGN KEY (det_id) +REFERENCES detRun(det_id), +FOREIGN KEY (exp_id) +REFERENCES rawExp(exp_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detProcessedImfile ( + det_id BIGINT, + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, exp_id, class_id), + KEY(fault), + INDEX(det_id, class_id), + INDEX(det_id, exp_id), + FOREIGN KEY (det_id, exp_id) + REFERENCES detInputExp(det_id, exp_id), + FOREIGN KEY (exp_id, class_id) + REFERENCES rawImfile(exp_id, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detProcessedExp ( + det_id BIGINT, + exp_id BIGINT, + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, exp_id), + KEY(fault), + FOREIGN KEY (det_id, exp_id) + REFERENCES detInputExp(det_id, exp_id), + FOREIGN KEY (det_id, exp_id) + REFERENCES detProcessedImfile(det_id, exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detStackedImfile ( + det_id BIGINT, + iteration INT, + class_id VARCHAR(64), + uri VARCHAR(255), + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault), + FOREIGN KEY (det_id, iteration) + REFERENCES detInputExp(det_id, iteration), + FOREIGN KEY (det_id, class_id) + REFERENCES detProcessedImfile(det_id, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detNormalizedStatImfile ( + det_id BIGINT, + iteration INT, + class_id VARCHAR(64), + norm FLOAT, + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault), + FOREIGN KEY (det_id, iteration) + REFERENCES detInputExp(det_id, iteration), + FOREIGN KEY (det_id, iteration, class_id) + REFERENCES detStackedImfile(det_id, iteration, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detNormalizedImfile ( + det_id BIGINT, + iteration INT, + class_id VARCHAR(64), + uri VARCHAR(255), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault), + INDEX(det_id, iteration), + FOREIGN KEY (det_id) + REFERENCES detInputExp(det_id), + FOREIGN KEY (det_id, iteration, class_id) + REFERENCES detNormalizedStatImfile(det_id, iteration, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detNormalizedExp ( + det_id BIGINT, + iteration INT, + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration), + KEY(fault), + FOREIGN KEY (det_id, iteration) + REFERENCES detInputExp(det_id, iteration), + FOREIGN KEY (det_id, iteration) + REFERENCES detNormalizedImfile(det_id, iteration) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detResidImfile ( + det_id BIGINT, + iteration INT, + ref_det_id BIGINT, + ref_iter INT, + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + bg_skewness DOUBLE, + bg_kurtosis DOUBLE, + bin_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + fringe_resid_0 DOUBLE, + fringe_resid_1 DOUBLE, + fringe_resid_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, exp_id, class_id), + KEY(fault), + INDEX(det_id, iteration, exp_id), + FOREIGN KEY (det_id, iteration, exp_id) + REFERENCES detInputExp(det_id, iteration, exp_id), + -- FOREIGN KEY (det_id, exp_id, class_id) + -- REFERENCES detProcessedImfile(det_id, exp_id, class_id), + FOREIGN KEY (ref_det_id, ref_iter) + REFERENCES detNormalizedExp(det_id, iteration) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detResidExp ( + det_id BIGINT, + iteration INT, + exp_id BIGINT, + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + bg_skewness DOUBLE, + bg_kurtosis DOUBLE, + bin_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + fringe_resid_0 DOUBLE, + fringe_resid_1 DOUBLE, + fringe_resid_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + accept TINYINT, + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, exp_id), + KEY(fault), + INDEX(det_id, iteration), + FOREIGN KEY (det_id, iteration, exp_id) + REFERENCES detInputExp(det_id, iteration, exp_id), + FOREIGN KEY (det_id, iteration, exp_id) + REFERENCES detResidImfile(det_id, iteration, exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detRunSummary ( + det_id BIGINT, + iteration INT, + data_state VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + accept TINYINT, + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration), + KEY(fault), + FOREIGN KEY (det_id, iteration) + REFERENCES detInputExp(det_id, iteration), + FOREIGN KEY (det_id, iteration) + REFERENCES detResidExp(det_id, iteration) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detRegisteredImfile ( + det_id BIGINT, + iteration INT, + class_id VARCHAR(64), + uri VARCHAR(255), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault), + FOREIGN KEY (det_id, iteration) + REFERENCES detRun(det_id, iteration) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpRun ( + warp_id BIGINT AUTO_INCREMENT, + fake_id BIGINT, + mode VARCHAR(64), + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + registered DATETIME, + magiced TINYINT, + PRIMARY KEY(warp_id), + KEY(warp_id), + KEY(fake_id), + KEY(mode), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(end_stage), + KEY(magiced), + INDEX(warp_id, fake_id), + FOREIGN KEY (fake_id) + REFERENCES fakeRun(fake_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpSkyCellMap ( + warp_id BIGINT, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + class_id VARCHAR(64), + fault SMALLINT, + PRIMARY KEY(warp_id, skycell_id, tess_id, class_id), + KEY(fault), + FOREIGN KEY (warp_id) + REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpSkyfile ( + warp_id BIGINT, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + uri VARCHAR(255), + path_base VARCHAR(255), + data_state VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + dtime_warp FLOAT, + hostname VARCHAR(64), + good_frac FLOAT, + xmin INT, + xmax INT, + ymin INT, + ymax INT, + ignored TINYINT, + fault SMALLINT, + PRIMARY KEY(warp_id, skycell_id, tess_id), + KEY(good_frac), + KEY(ignored), KEY(fault), + FOREIGN KEY (warp_id, skycell_id, tess_id) + REFERENCES warpSkyCellMap(warp_id, skycell_id, tess_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpMask ( + label VARCHAR(64), + PRIMARY KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackRun ( + stack_id BIGINT AUTO_INCREMENT, + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + reduction VARCHAR(64), + dvodb VARCHAR(255), + registered DATETIME, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + filter VARCHAR(64), + PRIMARY KEY(stack_id), + KEY(stack_id), + KEY(state), + KEY(skycell_id), + KEY(tess_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackInputSkyfile ( + stack_id BIGINT, + warp_id BIGINT, + PRIMARY KEY(stack_id, warp_id), + FOREIGN KEY (stack_id) REFERENCES stackRun(stack_id), + FOREIGN KEY (warp_id) REFERENCES warpSkyfile(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackSumSkyfile ( + stack_id BIGINT, + uri VARCHAR(255), + path_base VARCHAR(255), + data_state VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + dtime_stack FLOAT, + hostname VARCHAR(64), + good_frac FLOAT, + fault SMALLINT, + PRIMARY KEY(stack_id), + KEY(dtime_stack), + KEY(good_frac), + KEY(fault), + FOREIGN KEY (stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE diffRun ( + diff_id BIGINT AUTO_INCREMENT, + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + reduction VARCHAR(64), + dvodb VARCHAR(255), + registered DATETIME, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + PRIMARY KEY(diff_id), + KEY(diff_id), + KEY(state), + KEY(skycell_id), + KEY(tess_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE diffInputSkyfile ( + diff_id BIGINT, + template TINYINT, + stack_id BIGINT, + warp_id BIGINT, + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + kind VARCHAR(64), + PRIMARY KEY(diff_id, template), + KEY(stack_id), + KEY(warp_id), + KEY(skycell_id), + KEY(tess_id), + KEY(kind), + FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id), + FOREIGN KEY (stack_id) REFERENCES stackSumSkyfile(stack_id), + FOREIGN KEY (warp_id, skycell_id, tess_id) REFERENCES warpSkyfile(warp_id, skycell_id, tess_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE diffSkyfile ( + diff_id BIGINT, + uri VARCHAR(255), + path_base VARCHAR(255), + data_state VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + stamps_num INT, + stamps_rms FLOAT, + sources INT, + dtime_diff FLOAT, + hostname VARCHAR(64), + good_frac FLOAT, + fault SMALLINT, + PRIMARY KEY(diff_id), + KEY(good_frac), + KEY(fault), + FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicRun ( + magic_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(255), + label VARCHAR(64), + dvodb VARCHAR(255), + registered DATETIME, + fault SMALLINT, + PRIMARY KEY(magic_id), + KEY(magic_id), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(fault), + FOREIGN KEY (exp_id) REFERENCES rawExp(exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicInputSkyfile ( + magic_id BIGINT, + diff_id BIGINT, + node VARCHAR(64), + PRIMARY KEY(magic_id, diff_id), + FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), + FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicTree ( + magic_id BIGINT, + node VARCHAR(64), + dep VARCHAR(64), + KEY(magic_id), + KEY(node), + KEY(dep), + INDEX(magic_id, node), + FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicNodeResult ( + magic_id BIGINT, + node VARCHAR(64), + uri VARCHAR(255), + fault SMALLINT, + PRIMARY KEY(magic_id, node), + FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), + FOREIGN KEY (magic_id, node) REFERENCES magicTree(magic_id, node), + KEY(fault) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicMask ( + magic_id BIGINT, + uri VARCHAR(255), + streaks INT, + fault SMALLINT, + PRIMARY KEY(magic_id), + FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), + KEY(fault) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE calDB ( + cal_id BIGINT AUTO_INCREMENT, + dvodb VARCHAR(64), + state VARCHAR(64), + PRIMARY KEY(cal_id), + KEY(cal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE calRun ( + cal_id BIGINT AUTO_INCREMENT, + region VARCHAR(64), + last_step VARCHAR(64), + state VARCHAR(64), + PRIMARY KEY(cal_id), + KEY(cal_id), + KEY(last_step), + FOREIGN KEY (cal_id) REFERENCES calDB(cal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE flatcorrRun ( + corr_id BIGINT AUTO_INCREMENT, + dvodb VARCHAR(64), + filter VARCHAR(64), + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + reduction VARCHAR(64), + region VARCHAR(64), + hostname VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(corr_id), + KEY(corr_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- these two tables link the flatcorrRun to the associated chip and camera runs +CREATE TABLE flatcorrChipLink ( + corr_id BIGINT, + chip_id BIGINT, + PRIMARY KEY(corr_id, chip_id), + FOREIGN KEY (corr_id) REFERENCES flatcorrRun(corr_id), + FOREIGN KEY (chip_id) REFERENCES chipRun(chip_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE flatcorrCamLink ( + corr_id BIGINT, + chip_id BIGINT, + cam_id BIGINT, + PRIMARY KEY(corr_id, cam_id), + FOREIGN KEY (corr_id) REFERENCES flatcorrRun(corr_id), + FOREIGN KEY (chip_id) REFERENCES chipRun(chip_id), + FOREIGN KEY (cam_id) REFERENCES camRun(cam_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampDataStore ( + ds_id BIGINT AUTO_INCREMENT, + state VARCHAR(64), + lastFileset VARCHAR(64), + outProduct VARCHAR(64) UNIQUE, + uri VARCHAR(255), + PRIMARY KEY(ds_id), + KEY(ds_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampRequest ( + req_id BIGINT AUTO_INCREMENT, + ds_id BIGINT, + state VARCHAR(64), + name VARCHAR(64) UNIQUE, + reqType VARCHAR(16), + outProduct VARCHAR(64), + uri VARCHAR(255), + fault SMALLINT, + PRIMARY KEY(req_id), + KEY(req_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampJob ( + job_id BIGINT AUTO_INCREMENT, + req_id BIGINT, + rownum VARCHAR(64), + state VARCHAR(64), + jobType VARCHAR(16), + fault SMALLINT, + uri VARCHAR(255), + exp_id BIGINT, + outputBase VARCHAR(255), + args VARCHAR(511), + PRIMARY KEY(job_id, req_id), + KEY(job_id), + FOREIGN KEY (req_id) REFERENCES pstampRequest(req_id) +) ENGINE=innodb DEFAULT CHARSET=latin1 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/pxadmin_drop_tables.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/pxadmin_drop_tables.sql (revision 22161) @@ -0,0 +1,57 @@ +SET FOREIGN_KEY_CHECKS=0; +DROP TABLE IF EXISTS pzDataStore; +DROP TABLE IF EXISTS summitExp; +DROP TABLE IF EXISTS summitImfile; +DROP TABLE IF EXISTS pzDownloadExp; +DROP TABLE IF EXISTS pzDownloadImfile; +DROP TABLE IF EXISTS newExp; +DROP TABLE IF EXISTS newImfile; +DROP TABLE IF EXISTS rawExp; +DROP TABLE IF EXISTS rawImfile; +DROP TABLE IF EXISTS guidePendingExp; +DROP TABLE IF EXISTS chipRun; +DROP TABLE IF EXISTS chipProcessedImfile; +DROP TABLE IF EXISTS chipMask; +DROP TABLE IF EXISTS camRun; +DROP TABLE IF EXISTS camProcessedExp; +DROP TABLE IF EXISTS camMask; +DROP TABLE IF EXISTS detRun; +DROP TABLE IF EXISTS detInputExp; +DROP TABLE IF EXISTS fakeRun; +DROP TABLE IF EXISTS fakeProcessedImfile; +DROP TABLE IF EXISTS fakeMask; +DROP TABLE IF EXISTS detProcessedImfile; +DROP TABLE IF EXISTS detProcessedExp; +DROP TABLE IF EXISTS detStackedImfile; +DROP TABLE IF EXISTS detNormalizedStatImfile; +DROP TABLE IF EXISTS detNormalizedImfile; +DROP TABLE IF EXISTS detNormalizedExp; +DROP TABLE IF EXISTS detResidImfile; +DROP TABLE IF EXISTS detResidExp; +DROP TABLE IF EXISTS detRunSummary; +DROP TABLE IF EXISTS detRegisteredImfile; +DROP TABLE IF EXISTS warpRun; +DROP TABLE IF EXISTS warpSkyCellMap; +DROP TABLE IF EXISTS warpSkyfile; +DROP TABLE IF EXISTS warpMask; +DROP TABLE IF EXISTS diffRun; +DROP TABLE IF EXISTS diffInputSkyfile; +DROP TABLE IF EXISTS diffSkyfile; +DROP TABLE IF EXISTS stackRun; +DROP TABLE IF EXISTS stackInputSkyfile; +DROP TABLE IF EXISTS stackSumSkyfile; +DROP TABLE IF EXISTS magicRun; +DROP TABLE IF EXISTS magicInputSkyfile; +DROP TABLE IF EXISTS magicTree; +DROP TABLE IF EXISTS magicNodeResult; +DROP TABLE IF EXISTS magicMask; +DROP TABLE IF EXISTS magicSkyfileMask; +DROP TABLE IF EXISTS calDB; +DROP TABLE IF EXISTS calRun; +DROP TABLE IF EXISTS flatcorrRun; +DROP TABLE IF EXISTS flatcorrChipLink; +DROP TABLE IF EXISTS flatcorrCamLink; +DROP TABLE IF EXISTS pstampDataStore; +DROP TABLE IF EXISTS pstampRequest; +DROP TABLE IF EXISTS pstampJob; +SET FOREIGN_KEY_CHECKS=1 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/pztool_find_completed_exp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/pztool_find_completed_exp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/pztool_find_completed_exp.sql (revision 22161) @@ -0,0 +1,35 @@ +SELECT DISTINCT + exp_name, -- return should match pzDownloadExp + camera, + telescope, + state, + NULL as epoch -- epoch +FROM ( + SELECT + pzDownloadImfile.*, + pzDownloadExp.state, + summitExp.imfiles + FROM pzDownloadExp + JOIN summitExp + USING(exp_name, camera, telescope) + LEFT JOIN pzDownloadImfile + USING(exp_name, camera, telescope) + LEFT JOIN newExp + ON pzDownloadExp.exp_name = newExp.tmp_exp_name + AND pzDownloadExp.camera = newExp.tmp_camera + AND pzDownloadExp.telescope = newExp.tmp_telescope + WHERE + pzDownloadExp.state = 'run' + AND newExp.tmp_exp_name IS NULL + AND newExp.tmp_camera IS NULL + AND newExp.tmp_telescope IS NULL + GROUP BY + pzDownloadExp.exp_name, + pzDownloadExp.camera, + pzDownloadExp.telescope + -- it doesn't matter which field in pzDownloadImfile we count as we've + -- already Download a group by + HAVING + COUNT(pzDownloadImfile.exp_name) = summitExp.imfiles + AND SUM(pzDownloadImfile.fault) = 0 + ) as Foo Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/pztool_pendingimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/pztool_pendingimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/pztool_pendingimfile.sql (revision 22161) @@ -0,0 +1,20 @@ +SELECT * +FROM ( + SELECT + summitImfile.*, + summitExp.dateobs + FROM summitImfile + JOIN pzDownloadExp + USING(exp_name, camera, telescope) + LEFT JOIN pzDownloadImfile + USING(exp_name, camera, telescope, class, class_id) + JOIN summitExp + USING(exp_name, camera, telescope) + WHERE + pzDownloadExp.state = 'run' + AND pzDownloadImfile.exp_name IS NULL + AND pzDownloadImfile.camera IS NULL + AND pzDownloadImfile.telescope IS NULL + AND pzDownloadImfile.class IS NULL + AND pzDownloadImfile.class_id IS NULL +) as Foo Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/pztool_revert_downloadimfile_faults.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/pztool_revert_downloadimfile_faults.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/pztool_revert_downloadimfile_faults.sql (revision 22161) @@ -0,0 +1,17 @@ +DELETE FROM pzDownloadImfile +WHERE + fault != 0 + -- fault was logged in the last 3 days + AND ABS(TIMESTAMPDIFF(SECOND, epoch, NOW())) < 86400 * 3 + AND ( + -- HTTP 503: try again + fault = 203 + -- HTTP 500: timeout + OR fault = 200 + -- HTTP 404: unknown datastore internal problem + OR fault = 104 + -- ipptool errors + OR fault < 100 + -- perl untrapped die() + OR fault = 255 + ) Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/pztool_revert_fileset_faults.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/pztool_revert_fileset_faults.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/pztool_revert_fileset_faults.sql (revision 22161) @@ -0,0 +1,17 @@ +UPDATE summitExp SET fault = 0 +WHERE + fault != 0 + -- fault is older than 10 minutes + AND ABS(TIMESTAMPDIFF(SECOND, epoch, NOW())) > 60 * 10 + -- fault was logged in the last 3 days + AND ABS(TIMESTAMPDIFF(SECOND, epoch, NOW())) < 86400 * 3 + AND ( + -- HTTP 500: timeout + fault = 200 + -- HTTP 404: unknown datastore internal problem + OR fault = 104 + -- ipptool errors + OR fault < 100 + -- perl untrapped die() + OR fault = 255 + ) Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/pztool_revertcopied.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/pztool_revertcopied.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/pztool_revertcopied.sql (revision 22161) @@ -0,0 +1,3 @@ +DELETE FROM pzDownloadImfile +WHERE + fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_create_dup_table.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_create_dup_table.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_create_dup_table.sql (revision 22161) @@ -0,0 +1,2 @@ +CREATE TEMPORARY TABLE duplicate (exp_id BIGINT, PRIMARY KEY(exp_id)) +ENGINE=MEMORY Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_pendingexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_pendingexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_pendingexp.sql (revision 22161) @@ -0,0 +1,39 @@ +-- output should match the format of newExp +SELECT DISTINCT + exp_id, + tmp_exp_name, + tmp_camera, + tmp_telescope, + state, + workdir, + workdir_state, + reduction, + dvodb, + tess_id, + end_stage, + label, + epoch +FROM + (SELECT + newExp.*, + newImfile.tmp_class_id, + rawImfile.tmp_class_id as raw_tmp_class_id + FROM newExp + JOIN newImfile + USING(exp_id) + LEFT JOIN rawExp + USING(exp_id) + LEFT JOIN rawImfile + ON newImfile.exp_id = rawImfile.exp_id + AND newImfile.tmp_class_id = rawImfile.tmp_class_id + WHERE + newExp.state = 'run' + AND rawExp.exp_id IS NULL +-- where hook %s + GROUP BY + newExp.exp_id + HAVING + COUNT(newImfile.tmp_class_id) = COUNT(rawImfile.tmp_class_id) + AND SUM(rawImfile.fault) = 0 +-- limit hook %s + ) as Foo Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_pendingimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_pendingimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_pendingimfile.sql (revision 22161) @@ -0,0 +1,17 @@ +SELECT DISTINCT + newExp.exp_id, + newExp.tmp_exp_name, + newExp.tmp_camera, + newExp.tmp_telescope, + newExp.workdir, + newImfile.tmp_class_id, + newImfile.uri +FROM newImfile +JOIN newExp + USING(exp_id) +LEFT JOIN rawImfile + ON newExp.exp_id = rawImfile.exp_id + AND newImfile.tmp_class_id = rawImfile.tmp_class_id +WHERE + newExp.state = 'run' + AND rawImfile.tmp_class_id IS NULL Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_populate_dup_table.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_populate_dup_table.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_populate_dup_table.sql (revision 22161) @@ -0,0 +1,13 @@ +INSERT INTO duplicate +SELECT exp_id FROM + ( + SELECT + MAX(exp_id) AS exp_id, +-- tmp_exp_name, +-- tmp_camera, +-- tmp_telescope, + count(exp_id) AS count + FROM newExp + GROUP BY tmp_exp_name, tmp_camera, tmp_telescope + HAVING count > 1 + ) as Foo Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_processedimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_processedimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_processedimfile.sql (revision 22161) @@ -0,0 +1,10 @@ +SELECT + rawImfile.*, + newExp.tmp_exp_name, + newExp.tmp_camera, + newExp.tmp_telescope +FROM rawImfile +JOIN newExp + USING(exp_id) +-- bogus conditional so there is a where clause to append to +WHERE rawImfile.exp_id is NOT NULL Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_revertprocessedexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_revertprocessedexp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_revertprocessedexp.sql (revision 22161) @@ -0,0 +1,3 @@ +DELETE FROM rawExp +WHERE + fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_revertprocessedimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_revertprocessedimfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/regtool_revertprocessedimfile.sql (revision 22161) @@ -0,0 +1,3 @@ +DELETE FROM rawImfile +WHERE + fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_definebyquery_insert.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_definebyquery_insert.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_definebyquery_insert.sql (revision 22161) @@ -0,0 +1,23 @@ +-- Define the inputs to a defined stack +INSERT INTO + stackInputSkyfile(stack_id, warp_id) +SELECT + @STACK_ID@, -- This should be replaced with the stack_id + warpRun.warp_id +FROM warpSkyfile +JOIN warpRun + USING(warp_id) +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) +WHERE + warpSkyfile.skycell_id = '%s' + AND warpRun.state = 'full' + AND rawExp.filter = '%s' + AND warpSkyfile.fault = 0 + AND warpSkyfile.ignored = 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_definebyquery_insert_random_part1.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_definebyquery_insert_random_part1.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_definebyquery_insert_random_part1.sql (revision 22161) @@ -0,0 +1,32 @@ +-- This is the part 1 of 2 of a query to use a random set of warps for +-- the inputs to a defined stack. +-- stacktool_definebyquery_insert_random_part2.sql should be appended. +INSERT INTO + stackInputSkyfile(stack_id, warp_id) +SELECT + @STACK_ID@, -- This should be replaced with the stack_id + warp_id +FROM ( + -- Sub-select to get the random set of warps + SELECT + warpSkyfile.*, + rand() AS rnd_num + FROM warpSkyfile + JOIN warpRun + USING(warp_id) + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + WHERE + skycell_id = '%s' + AND warpRun.state = 'full' + AND rawExp.filter = '%s' -- the result of the query is grouped by filter and inserted for one at a time + AND warpSkyfile.fault = 0 + AND warpSkyfile.ignored = 0 +-- Put additional constraints here +-- stacktool_definebyquery_insert_random_part2.sql goes here Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_definebyquery_insert_random_part2.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_definebyquery_insert_random_part2.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_definebyquery_insert_random_part2.sql (revision 22161) @@ -0,0 +1,8 @@ +-- This is the part 2 of 2 of a query to use a random set of warps for +-- the inputs to a defined stack. +-- stacktool_definebyquery_insert_random_part1.sql should be prepended. + -- Sort by the random number, and take the first N + -- to get a random set of N. + ORDER BY rnd_num + LIMIT %d +) AS randomWarps Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_definebyquery_part1.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_definebyquery_part1.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_definebyquery_part1.sql (revision 22161) @@ -0,0 +1,30 @@ +-- This is the first part of the query to get a list of skycells with +-- warps to be stacked, along with the number of warps already in a +-- stack. It needs to be completed by part 2 (see below). + +SELECT + skycell_id, + filter, + tess_id, + num_warp, + MAX(num_stack) AS num_stack +FROM (( + -- Number of stack-ready warps as a function of skycell and filter + SELECT + skycell_id, + warpSkyfile.tess_id as tess_id, + rawExp.filter, + COUNT(warpSkyfile.skycell_id) AS num_warp -- number of warps that can be stacked + FROM warpRun + JOIN warpSkyfile USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + WHERE + warpRun.state = 'full' + AND warpSkyfile.ignored = 0 + AND warpSkyfile.fault = 0 + -- Here should follow the SQL in stacktool_definebyquery_part2.sql, + -- after any additional WHERE conditions have been added + Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_definebyquery_part2.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_definebyquery_part2.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_definebyquery_part2.sql (revision 22161) @@ -0,0 +1,28 @@ + +-- This is the second part of the query to get a list of skycells with +-- warps to be stacked, along with the number of warps already in a +-- stack. It should follow part 1, which is in +-- stacktool_definebyquery_part1.sql + + GROUP BY + skycell_id, + filter + ) AS warpsToStack + LEFT JOIN ( + -- Number of stack inputs as a function of skycell and filter + SELECT + skycell_id, + stackRun.tess_id as stack_tess_id, + filter, + COUNT(stackInputSkyfile.warp_id) as num_stack -- number of warps in a stack + FROM stackRun + JOIN stackInputSkyfile USING(stack_id) + GROUP BY + stack_id + ) AS stackSizes + -- JOINing the warpsToStack and stackSizes tables + USING(skycell_id, filter) + ) + GROUP BY + skycell_id, + filter Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_definebyquery_test.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_definebyquery_test.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_definebyquery_test.sql (revision 22161) @@ -0,0 +1,59 @@ +-- This is the combination of the two parts of the query to get a list +-- of skycells with warps to be stacked, along with the number of +-- warps already in a stack. + +-------------------------------------------------------------------------------- +-- THIS FILE IS INTENDED FOR TESTING ONLY! IT IS NOT USED BY stacktool. +-- CHANGES SHOULD BE MADE TO stacktool_definebyquery_part1.sql +-- AND stacktool_definebyquery_part2.sql +-------------------------------------------------------------------------------- + +-- stacktool_definebyquery_part1.sql +SELECT + skycell_id, + tess_id, + filter, + num_warp, + MAX(num_stack) AS num_stack +FROM (( + -- Number of stack-ready warps as a function of skycell and filter + SELECT + skycell_id, + tess_id, + filter, + COUNT(warpSkyfile.skycell_id) AS num_warp -- number of warps that can be stacked + FROM warpRun + JOIN warpSkyfile USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + WHERE + warpRun.state = 'full' + AND warpSkyfile.ignored = 0 + AND warpSkyfile.fault = 0 + -- Any additional selection on warps/exposures goes here +-- stacktool_definebyquery_part2.sql + GROUP BY + skycell_id, + filter + ) AS warpsToStack + LEFT JOIN ( + -- Number of stack inputs as a function of skycell and filter + SELECT + skycell_id, + tess_id, + filter, + COUNT(stackInputSkyfile.warp_id) as num_stack -- number of warps in a stack + FROM stackRun + JOIN stackInputSkyfile USING(stack_id) + GROUP BY + stack_id + ) AS stackSizes + -- JOINing the warpsToStack and stackSizes tables + USING(skycell_id, filter) + ) + GROUP BY + skycell_id, + filter +; Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_donecleanup.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_donecleanup.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_donecleanup.sql (revision 22161) @@ -0,0 +1,5 @@ +SELECT + stackRun.* +FROM stackRun +WHERE + stackRun.state = 'cleaned' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_find_complete_warps.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_find_complete_warps.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_find_complete_warps.sql (revision 22161) @@ -0,0 +1,45 @@ +SELECT + warpSkyfile.skycell_id, + warpSkyfile.tess_id, + stackRun.stack_id, + COUNT(warpSkyfile.skycell_id) AS num_avail, + COUNT(stackRun.stack_id) AS num_extant +FROM warpRun +JOIN warpSkyfile + USING(warp_id) +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) +LEFT JOIN stackInputSkyfile + ON warpSkyfile.warp_id = stackInputSkyfile.warp_id +LEFT JOIN stackRun + ON stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.stack_id = stackInputSkyfile.stack_id +WHERE + warpRun.state = 'full' + AND warpSkyfile.ignored = 0 +GROUP BY + warpSkyfile.skycell_id, stackRun.stack_id +HAVING + num_avail > num_extant + +-- It seems like we should be grouping the results here but in fact we don't +-- want to do that as each warp_id may contain multiple skycells. So a warp_id +-- may end up in more than one stackRun (for different skycells) +-- GROUP BY +-- warp_id + + +INSERT INTO + stackInputSkyfile(stack_id, warp_id) +SELECT + 13, + warp_id +FROM warpSkyfile +WHERE + skycell_id = 'new'; Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_inputskyfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_inputskyfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_inputskyfile.sql (revision 22161) @@ -0,0 +1,22 @@ +SELECT DISTINCT + warpSkyfile.*, + rawExp.camera +FROM stackRun +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpSkyfile + ON stackInputSkyfile.warp_id = warpSkyfile.warp_id + AND stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.tess_id = warpSkyfile.tess_id +JOIN warpRun + ON stackInputSkyfile.warp_id = warpRun.warp_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + USING(chip_id) +JOIN rawExp + ON chipProcessedImfile.exp_id = rawExp.exp_id Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_pendingcleanuprun.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_pendingcleanuprun.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_pendingcleanuprun.sql (revision 22161) @@ -0,0 +1,29 @@ +-- results in too many runs adding DISTINCT fixes this +-- is the camera needed for this? YES I think so that update can define the camera +-- is the camera unique for this? YES. Well, sort of. +-- stack_skycell.pl insures that the files +-- we're here to clean come from the same camera so +-- if the camera is not unique we have nothing to do +SELECT DISTINCT + stackRun.stack_id, + rawExp.camera, + stackRun.state +FROM stackRun +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpSkyfile + ON stackInputSkyfile.warp_id = warpSkyfile.warp_id + AND stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.tess_id = warpSkyfile.tess_id +JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING (exp_id) +WHERE + stackRun.state = 'goto_cleaned' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_pendingcleanupskyfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_pendingcleanupskyfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_pendingcleanupskyfile.sql (revision 22161) @@ -0,0 +1,11 @@ +SELECT + stackSumSkyfile.*, + stackRun.state, + stackRun.workdir, + stackRun.dvodb, + stackRun.tess_id +FROM stackRun +JOIN stackSumSkyfile + USING(stack_id) +WHERE + stackRun.state = 'goto_cleaned' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_revertsumskyfile_delete.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_revertsumskyfile_delete.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_revertsumskyfile_delete.sql (revision 22161) @@ -0,0 +1,4 @@ +DELETE FROM stackSumSkyfile +USING stackSumSkyfile, stackRun +WHERE stackSumSkyfile.stack_id = stackRun.stack_id + AND fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_revertsumskyfile_update.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_revertsumskyfile_update.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_revertsumskyfile_update.sql (revision 22161) @@ -0,0 +1,5 @@ +UPDATE stackRun +JOIN stackSumSkyfile USING(stack_id) +SET stackRun.state = 'new' +WHERE + fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_sumskyfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_sumskyfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_sumskyfile.sql (revision 22161) @@ -0,0 +1,19 @@ +SELECT + stackSumSkyfile.*, + stackRun.state, + rawExp.camera +FROM stackRun +JOIN stackSumSkyfile + USING(stack_id) +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpRun + USING(warp_id) +JOIN fakeRun + ON warpRun.fake_id = fakeRun.fake_id +JOIN camRun + ON camRun.cam_id = fakeRun.cam_id +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_tosum.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_tosum.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/stacktool_tosum.sql (revision 22161) @@ -0,0 +1,18 @@ +SELECT DISTINCT + stackRun.stack_id, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.workdir, + stackRun.label, + stackRun.state +FROM stackRun +JOIN stackInputSkyfile + USING(stack_id) +LEFT JOIN stackSumSkyfile + USING(stack_id) +WHERE + ((stackRun.state = 'new' + AND stackSumSkyfile.stack_id IS NULL) + OR + (stackRun.state = 'update' + AND stackSumSkyfile.data_state = 'cleaned')) Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_change_run_state.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_change_run_state.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_change_run_state.sql (revision 22161) @@ -0,0 +1,15 @@ +-- change state of warpRun from goto_cleaned to cleaned or goto_purged to purged +-- when all of the consituant skyfiles are in the end state +-- arguments are new state (cleaned or purged) warp_id and new state again for +-- the chipProcessedImfile sub query +UPDATE warpRun + SET state = '%s' + WHERE + warpRun.warp_id = %lld + AND (SELECT + COUNT(warp_id) + FROM warpSkyfile + WHERE + warpSkyfile.warp_id = warpRun.warp_id + AND data_state != '%s' + ) = 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_change_skyfile_data_state.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_change_skyfile_data_state.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_change_skyfile_data_state.sql (revision 22161) @@ -0,0 +1,15 @@ +-- handle changes in warpSkyfile.data_state. +-- Used for the modes tocleanedskyfile and topurgedskyfile +-- args are new data_state, warp_id, skycell_id and current expected state for warpRun +UPDATE warpSkyfile + SET + data_state = '%s' +WHERE + warp_id = %lld + AND skycell_id = '%s' + -- only update if chipRun.state has the expected value + AND ( + SELECT state from warpRun where warpRun.warp_id = warpSkyfile.warp_id + ) = '%s' + + Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_definebyquery.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_definebyquery.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_definebyquery.sql (revision 22161) @@ -0,0 +1,36 @@ +SELECT + * +FROM + (SELECT DISTINCT + fakeRun.*, + chipRun.chip_id, + rawExp.camera, + rawExp.telescope, + rawExp.dateobs, + rawExp.exp_tag, + rawExp.exp_type, + rawExp.filelevel, + rawExp.filter, + rawExp.airmass, + rawExp.ra, + rawExp.decl, + rawExp.exp_time, + rawExp.sat_pixel_frac, + rawExp.bg, + rawExp.bg_stdev, + rawExp.bg_mean_stdev, + rawExp.alt, + rawExp.az, + rawExp.ccd_temp, + rawExp.posang, + rawExp.object, + rawExp.solang + FROM fakeRun + JOIN camRun + using(cam_id) + JOIN chipRun + using(chip_id) + JOIN rawExp + using(exp_id) + WHERE + fakeRun.state = 'full') as Foo Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_donecleanup.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_donecleanup.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_donecleanup.sql (revision 22161) @@ -0,0 +1,5 @@ +SELECT + warpRun.* +FROM warpRun +WHERE + warpRun.state = 'cleaned' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_exp.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_exp.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_exp.sql (revision 22161) @@ -0,0 +1,8 @@ +SELECT + fakeRun.* +FROM warpRun +JOIN fakeRun + USING(fake_id) +WHERE + warpRun.state = 'new' + AND fakeRun.state = 'full' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_imfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_imfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_imfile.sql (revision 22161) @@ -0,0 +1,27 @@ +SELECT DISTINCT + rawImfile.*, + warpRun.fake_id, + camRun.cam_id as cam_id, + chipProcessedImfile.uri as chip_uri, + chipProcessedImfile.path_base as chip_path_base, + camProcessedExp.path_base as cam_path_base +FROM warpRun +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN camProcessedExp + USING(cam_id) +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN chipProcessedImfile + ON chipRun.chip_id = chipProcessedImfile.chip_id +JOIN rawImfile -- is there any reason not to refer back to rawimfiles? + ON chipProcessedImfile.exp_id = rawImfile.exp_id + AND chipProcessedImfile.class_id = rawImfile.class_id +WHERE + warpRun.state = 'new' + AND fakeRun.state = 'full' + AND camRun.state = 'full' + AND chipRun.state = 'full' + Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_pendingcleanuprun.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_pendingcleanuprun.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_pendingcleanuprun.sql (revision 22161) @@ -0,0 +1,15 @@ +SELECT + warpRun.warp_id, + rawExp.camera, + warpRun.state +FROM warpRun +JOIN fakeRun +USING (fake_id) +JOIN camRun +USING (cam_id) +JOIN chipRun +USING (chip_id) +JOIN rawExp +USING (exp_id) +WHERE + (warpRun.state = 'goto_cleaned' OR warpRun.state = 'goto_purged') Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_pendingcleanupskyfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_pendingcleanupskyfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_pendingcleanupskyfile.sql (revision 22161) @@ -0,0 +1,14 @@ +SELECT + warpSkyfile.*, + warpRun.state, + warpRun.workdir, + warpRun.label, + warpRun.dvodb, + warpRun.end_stage +FROM warpRun +JOIN warpSkyfile + USING(warp_id) +WHERE + (warpRun.state = 'goto_cleaned' AND warpSkyfile.data_state = 'full') + OR + (warpRun.state = 'goto_purged' AND warpSkyfile.data_state != 'purged') Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_revertwarped.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_revertwarped.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_revertwarped.sql (revision 22161) @@ -0,0 +1,9 @@ +DELETE FROM warpSkyfile +USING warpSkyfile, warpRun, fakeRun, camRun, chipRun, rawExp +WHERE + warpSkyfile.warp_id = warpRun.warp_id + AND warpRun.fake_id = fakeRun.fake_id + AND fakeRun.cam_id = camRun.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND warpSkyfile.fault != 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_scmap.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_scmap.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_scmap.sql (revision 22161) @@ -0,0 +1,24 @@ +SELECT + warpSkyCellMap.*, + chipProcessedImfile.uri, + chipProcessedImfile.path_base as chip_path_base, + camProcessedExp.path_base as cam_path_base +FROM warpRun +JOIN warpSkyCellMap + USING(warp_id) +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN camProcessedExp + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + ON chipRun.chip_id = chipProcessedImfile.chip_id + AND warpSkyCellMap.class_id = chipProcessedImfile.class_id +WHERE +-- warpRun.state = 'new' + fakeRun.state = 'full' + AND camRun.state = 'full' + AND chipRun.state = 'full' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_tooverlap.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_tooverlap.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_tooverlap.sql (revision 22161) @@ -0,0 +1,30 @@ +SELECT + warpRun.warp_id, + warpRun.fake_id, + warpRun.workdir, + warpRun.tess_id, + warpRun.label, + rawExp.camera, + exp_id, + exp_tag, + warpRun.magiced +FROM warpRun +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) +LEFT JOIN warpSkyCellMap + USING(warp_id) +LEFT JOIN warpMask + ON warpRun.label = warpMask.label +WHERE + warpRun.state = 'new' + AND fakeRun.state = 'full' + AND camRun.state = 'full' + AND chipRun.state = 'full' + AND warpSkyCellMap.warp_id IS NULL + AND warpMask.label IS NULL Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_towarped.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_towarped.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_towarped.sql (revision 22161) @@ -0,0 +1,42 @@ +SELECT DISTINCT + warpSkyCellMap.warp_id, + warpSkyCellMap.skycell_id, + warpSkyCellMap.tess_id, + warpRun.fake_id, + warpRun.state, + camRun.cam_id, + rawExp.camera, + rawExp.exp_tag, + warpRun.workdir +FROM warpRun +JOIN warpSkyCellMap + USING(warp_id) +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + USING(chip_id) +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id +LEFT JOIN warpSkyfile + ON warpRun.warp_id = warpSkyfile.warp_id + AND warpSkyCellMap.skycell_id = warpSkyfile.skycell_id + AND warpSkyCellMap.tess_id = warpSkyfile.tess_id +LEFT JOIN warpMask + ON warpRun.label = warpMask.label +WHERE + ((warpRun.state = 'new' + AND warpSkyfile.warp_id IS NULL + AND warpSkyfile.skycell_id IS NULL + AND warpSkyfile.tess_id IS NULL) + OR (warpRun.state = 'update' + AND warpSkyfile.data_state = 'cleaned') + ) + AND fakeRun.state = 'full' + AND camRun.state = 'full' + AND chipRun.state = 'full' + AND warpMask.label IS NULL + AND warpSkyCellMap.fault = 0 Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_updateskyfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_updateskyfile.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_updateskyfile.sql (revision 22161) @@ -0,0 +1,1 @@ +UPDATE warpSkyfile SET fault = %d WHERE warp_id = %lld AND skycell_id = '%s' Index: /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_warped.sql =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_warped.sql (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/share/warptool_warped.sql (revision 22161) @@ -0,0 +1,20 @@ +SELECT + warpSkyfile.*, + warpRun.state, + rawExp.exp_id, + rawExp.camera +FROM warpRun +JOIN warpSkyfile + USING(warp_id) +JOIN fakeRun + ON warpRun.fake_id = fakeRun.fake_id +JOIN camRun + ON camRun.cam_id = fakeRun.cam_id +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id +WHERE +-- bogus test; just here so there there is a 'WHERE' stmt to append conditionals too +-- XXX EAM : not needed: fix warptool.c + warpRun.warp_id is NOT NULL Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/.cvsignore =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/.cvsignore (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/.cvsignore (revision 22161) @@ -0,0 +1,32 @@ +.deps +.gdb_history +.libs +Makefile +Makefile.in +config.h +config.h.in +stamp-h1 +*.la +*.lo +pxtoolsErrorCodes.c +pxtoolsErrorCodes.h +pxadmin +pxinject +pztool +pzgetexp +pzgetimfiles +regtool +guidetool +chiptool +camtool +warptool +difftool +stacktool +faketool +dettool +detselect +pxdata.c +magictool +caltool +flatcorr +pstamptool Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/Makefile.am =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/Makefile.am (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/Makefile.am (revision 22161) @@ -0,0 +1,220 @@ +bin_PROGRAMS = \ + caltool \ + camtool \ + chiptool \ + detselect \ + dettool \ + difftool \ + flatcorr \ + magictool \ + pstamptool \ + pxadmin \ + pxinject \ + faketool \ + pzgetexp \ + pzgetimfiles \ + pztool \ + regtool \ + stacktool \ + warptool + + +bin_SCRIPTS = \ + fakemagic + +pkginclude_HEADERS = \ + pxadmin.h \ + pxcam.h \ + pxchip.h \ + pxconfig.h \ + pxdata.h \ + pxfake.h \ + pxregister.h \ + pxtag.h \ + pxtools.h \ + pxtoolsErrorCodes.h \ + pxtree.h \ + pxwarp.h + +noinst_HEADERS = \ + caltool.h \ + camtool.h \ + chiptool.h \ + detselect.h \ + dettool.h \ + difftool.h \ + flatcorr.h \ + faketool.h \ + magictool.h \ + pstamptool.h \ + pxinject.h \ + pzgetexp.h \ + pzgetimfiles.h \ + pztool.h \ + regtool.h \ + stacktool.h \ + warptool.h + +lib_LTLIBRARIES = libpxtools.la +libpxtools_la_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +libpxtools_la_LIBS = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) +libpxtools_la_LDFLAGS = -release $(PACKAGE_VERSION) +libpxtools_la_SOURCES = \ + pxcam.c \ + pxchip.c \ + pxconfig.c \ + pxdata.c \ + pxerrors.c \ + pxfake.c \ + pxfault.c \ + pxregister.c \ + pxtag.c \ + pxtools.c \ + pxtoolsErrorCodes.c \ + pxtree.c \ + pxwarp.c + +# for pxtools.h +AM_CPPFLAGS = -I$(top_srcdir)/src$ + +pztool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pztool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pztool_SOURCES = \ + pztool.c \ + pztoolConfig.c + +pstamptool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pstamptool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pstamptool_SOURCES = \ + pstamptool.c \ + pstamptoolConfig.c + +caltool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +caltool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +caltool_SOURCES = \ + caltool.c \ + caltoolConfig.c + +flatcorr_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +flatcorr_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +flatcorr_SOURCES = \ + flatcorr.c \ + flatcorrConfig.c + +regtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +regtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +regtool_SOURCES = \ + regtool.c \ + regtoolConfig.c + +chiptool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +chiptool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +chiptool_SOURCES = \ + chiptool.c \ + chiptoolConfig.c + +camtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +camtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +camtool_SOURCES = \ + camtool.c \ + camtoolConfig.c + +faketool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +faketool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +faketool_SOURCES = \ + faketool.c \ + faketoolConfig.c + +magictool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +magictool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +magictool_SOURCES = \ + magictool.c \ + magictoolConfig.c + +warptool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +warptool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +warptool_SOURCES = \ + warptool.c \ + warptoolConfig.c + +difftool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +difftool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +difftool_SOURCES = \ + difftool.c \ + difftoolConfig.c + +stacktool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +stacktool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +stacktool_SOURCES = \ + stacktool.c \ + stacktoolConfig.c + +pxadmin_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pxadmin_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pxadmin_SOURCES = \ + pxadmin.c \ + pxadminConfig.c + +pzgetexp_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pzgetexp_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pzgetexp_SOURCES = \ + pzgetexp.c \ + pzgetexpConfig.c + +pzgetimfiles_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pzgetimfiles_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pzgetimfiles_SOURCES = \ + pzgetimfiles.c \ + pzgetimfilesConfig.c + +dettool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +dettool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +dettool_SOURCES = \ + dettool.c \ + dettoolConfig.c \ + dettool_correction.c \ + dettool_detrunsummary.c \ + dettool_normalizedexp.c \ + dettool_normalizedimfile.c \ + dettool_normalizedstat.c \ + dettool_processedexp.c \ + dettool_processedimfile.c \ + dettool_residexp.c \ + dettool_residimfile.c \ + dettool_stack.c + +detselect_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +detselect_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +detselect_SOURCES = \ + detselect.c \ + detselectConfig.c + +pxinject_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pxinject_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pxinject_SOURCES = \ + pxinject.c \ + pxinjectConfig.c + +clean-local: + -rm -f TAGS +# Tags for emacs +tags: + etags `find . -name \*.[ch] -print` + +# Error codes. +BUILT_SOURCES = pxtoolsErrorCodes.h pxtoolsErrorCodes.c +CLEANFILES = pxtoolsErrorCodes.h pxtoolsErrorCodes.c +EXTRA_DIST = pxtoolsErrorCodes.dat pxtoolsErrorCodes.c.in pxtoolsErrorCodes.h.in fakemagic + +pxtoolsErrorCodes.h : pxtoolsErrorCodes.dat pxtoolsErrorCodes.h.in + $(ERRORCODES) --data=pxtoolsErrorCodes.dat --outdir=. pxtoolsErrorCodes.h + +pxtoolsErrorCodes.c : pxtoolsErrorCodes.dat pxtoolsErrorCodes.c.in pxtoolsErrorCodes.h + $(ERRORCODES) --data=pxtoolsErrorCodes.dat --outdir=. pxtoolsErrorCodes.c + +BUILT_SOURCES += pxdata.c +EXTRA_DIST += pxdata.c.template + +pxdata.c: $(srcdir)/pxdata.c.template $(srcdir)/Makefile + $(PERL) -pe 's|\@PKGDATADIR\@|$(pkgdatadir)|' $< > $@ + Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/caltool.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/caltool.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/caltool.c (revision 22161) @@ -0,0 +1,261 @@ +/* + * caltool.c + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include + +#include "pxtools.h" +#include "caltool.h" + +static bool adddbMode(pxConfig *config); +static bool dbsMode(pxConfig *config); +static bool addrunMode(pxConfig *config); +static bool runsMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = caltoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(CALTOOL_MODE_ADDDB, adddbMode); + MODECASE(CALTOOL_MODE_DBS, dbsMode); + MODECASE(CALTOOL_MODE_ADDRUN, addrunMode); + MODECASE(CALTOOL_MODE_RUNS, runsMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool adddbMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", true, false); + + if (!calDBInsert(config->dbh, + 0, // cal_id + dvodb, + "active" // state + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + + } + + return false; +} + + +static bool dbsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM calDB"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("caltool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "calDB", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool addrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(cal_id, config->args, "-cal_id", true, false); + PXOPT_LOOKUP_STR(region, config->args, "-region", true, false); + PXOPT_LOOKUP_STR(last_step, config->args, "-last_step", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + + if (!calRunInsert(config->dbh, + (psS64)atoll(cal_id), + region, + last_step, + state + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + + } + + return false; +} + + +static bool runsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-cal_id", "cal_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM calRun"); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "WHERE fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "WHERE fault = 0"); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " AND %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("caltool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "calRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/caltool.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/caltool.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/caltool.h (revision 22161) @@ -0,0 +1,35 @@ +/* + * caltool.h + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef CALTOOL_H +#define CALTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + CALTOOL_MODE_NONE = PXTOOL_MODE_NONE, + CALTOOL_MODE_DBS, + CALTOOL_MODE_ADDDB, + CALTOOL_MODE_ADDRUN, + CALTOOL_MODE_RUNS +} caltoolMode; + +pxConfig *caltoolConfig(pxConfig *config, int argc, char **argv); + +#endif // CALTOOL_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/caltoolConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/caltoolConfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/caltoolConfig.c (revision 22161) @@ -0,0 +1,104 @@ +/* + * caltoolConfig.c + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "caltool.h" + +pxConfig *caltoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *time = psTimeGetNow(PS_TIME_TAI); + psString now = psTimeToISO(time); + psFree(time); + + // -adddb + psMetadata *adddbArgs = psMetadataAlloc(); + psMetadataAddStr(adddbArgs, PS_LIST_TAIL, "-dvodb", 0, "define DVO db (required)", NULL); + + // -dbs + psMetadata *dbsArgs = psMetadataAlloc(); + psMetadataAddU64(dbsArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(dbsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addrun + psMetadata *addrunArgs = psMetadataAlloc(); + psMetadataAddStr(addrunArgs, PS_LIST_TAIL, "-cal_id", 0, "define calibration DB ID (required)", NULL); + psMetadataAddStr(addrunArgs, PS_LIST_TAIL, "-region", 0, "define region (required)", NULL); + psMetadataAddStr(addrunArgs, PS_LIST_TAIL, "-last_step", 0, "define last step (required)", NULL); + psMetadataAddStr(addrunArgs, PS_LIST_TAIL, "-state", 0, "define state (required)", NULL); + + // -runs + psMetadata *runsArgs = psMetadataAlloc(); + psMetadataAddStr(runsArgs, PS_LIST_TAIL, "-cal_id", 0, "search for calibration ID", NULL); + psMetadataAddU64(runsArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(runsArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(runsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-adddb", "add a DVO calibration DB", CALTOOL_MODE_ADDDB, adddbArgs); + PXOPT_ADD_MODE("-dbs", "list DVO calibration DBs", CALTOOL_MODE_DBS, dbsArgs); + PXOPT_ADD_MODE("-addrun", "add the results of a calibration run", CALTOOL_MODE_ADDRUN, addrunArgs); + PXOPT_ADD_MODE("-runs", "list the results of calibration runs", CALTOOL_MODE_RUNS, runsArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/camtool.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/camtool.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/camtool.c (revision 22161) @@ -0,0 +1,1001 @@ +/* + * camtool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxcam.h" +#include "camtool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool pendingexpMode(pxConfig *config); +static bool pendingimfileMode(pxConfig *config); +static bool addprocessedexpMode(pxConfig *config); +static bool processedexpMode(pxConfig *config); +static bool revertprocessedexpMode(pxConfig *config); +static bool updateprocessedexpMode(pxConfig *config); +static bool blockMode(pxConfig *config); +static bool maskedMode(pxConfig *config); +static bool unblockMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupexpMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = camtoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(CAMTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(CAMTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(CAMTOOL_MODE_PENDINGEXP, pendingexpMode); + MODECASE(CAMTOOL_MODE_PENDINGIMFILE, pendingimfileMode); + MODECASE(CAMTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpMode); + MODECASE(CAMTOOL_MODE_PROCESSEDEXP, processedexpMode); + MODECASE(CAMTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); + MODECASE(CAMTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + MODECASE(CAMTOOL_MODE_BLOCK, blockMode); + MODECASE(CAMTOOL_MODE_MASKED, maskedMode); + MODECASE(CAMTOOL_MODE_UNBLOCK, unblockMode); + MODECASE(CAMTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); + MODECASE(CAMTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpMode); + MODECASE(CAMTOOL_MODE_DONECLEANUP, donecleanupMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + pxcamGetSearchArgs (config, where); + PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("camtool_find_chip_id.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, "%s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // would could do this "all in the database" if we didn't want the option + // of changing the label/reduction/expgroup/dvodb/etc. So we're pulling the + // data out so we have the option of changing these values or leaving the + // old values in place (i.e., passing the values through). + + // loop over our list of chipRun rows + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + chipRunRow *row = chipRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxcamQueueByChipID(config, + row->chip_id, + workdir ? workdir : row->workdir, + label ? label : row->label, + reduction ? reduction : row->reduction, + expgroup ? expgroup : row->expgroup, + dvodb ? dvodb : row->dvodb, + tess_id ? tess_id : row->tess_id, + end_stage ? end_stage : row->end_stage + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue chip_id: %" PRId64, row->chip_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + pxcamGetSearchArgs (config, where); + PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + if ((!state) && (!label)) { + psError(PXTOOLS_ERR_DATA, false, "parameters are required"); + psFree(where); + return false; + } + + if (state) { + // set chipRun.state to state + if (!pxcamRunSetStateByQuery(config, where, state)) { + psFree(where); + return false; + } + } + + if (label) { + // set chipRun.label to label + if (!pxcamRunSetLabelByQuery(config, where, label)) { + psFree(where); + return false; + } + } + + psFree(where); + + return true; +} + + +static bool pendingexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + pxcamGetSearchArgs (config, where); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("camtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, "%s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "camPendingExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool pendingimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + pxcamGetSearchArgs (config, where); + PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "=="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("camtool_find_pendingimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, "%s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipProcessedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", true, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + + // optional + PXOPT_LOOKUP_F32(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F32(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F32(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F32(bias, config->args, "-bias", false, false); + PXOPT_LOOKUP_F32(bias_stdev, config->args, "-bias_stdev", false, false); + PXOPT_LOOKUP_F32(fringe_0, config->args, "-fringe_0", false, false); + PXOPT_LOOKUP_F32(fringe_1, config->args, "-fringe_1", false, false); + PXOPT_LOOKUP_F32(fringe_2, config->args, "-fringe_2", false, false); + PXOPT_LOOKUP_F32(sigma_ra, config->args, "-sigma_ra", false, false); + PXOPT_LOOKUP_F32(sigma_dec, config->args, "-sigma_dec", false, false); + PXOPT_LOOKUP_F32(ap_resid, config->args, "-ap_resid", false, false); + PXOPT_LOOKUP_F32(ap_resid_stdev, config->args, "-ap_resid_stdev", false, false); + PXOPT_LOOKUP_F32(zp_mean, config->args, "-zp_mean", false, false); + PXOPT_LOOKUP_F32(zp_stdev, config->args, "-zp_stdev", false, false); + PXOPT_LOOKUP_F32(fwhm_major, config->args, "-fwhm_major", false, false); + PXOPT_LOOKUP_F32(fwhm_minor, config->args, "-fwhm_minor", false, false); + PXOPT_LOOKUP_F32(dtime_detrend, config->args, "-dtime_detrend", false, false); + PXOPT_LOOKUP_F32(dtime_photom, config->args, "-dtime_photom", false, false); + PXOPT_LOOKUP_F32(dtime_astrom, config->args, "-dtime_astrom", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S32(n_stars, config->args, "-n_stars", false, false); + PXOPT_LOOKUP_S32(n_extended, config->args, "-n_extended", false, false); + PXOPT_LOOKUP_S32(n_cr, config->args, "-n_cr", false, false); + PXOPT_LOOKUP_S32(n_astrom, config->args, "-n_astrom", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + // generate restrictions + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + + psString query = pxDataGet("camtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClaus = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, "%s", whereClaus); + psFree(whereClaus); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + camRunRow *pendingRow = camRunObjectFromMetadata(output->data[0]); + psFree(output); + camProcessedExpRow *row = camProcessedExpRowAlloc( + pendingRow->cam_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + bias, + bias_stdev, + fringe_0, + fringe_1, + fringe_2, + sigma_ra, + sigma_dec, + ap_resid, + ap_resid_stdev, + zp_mean, + zp_stdev, + fwhm_major, + fwhm_minor, + dtime_detrend, + dtime_photom, + dtime_astrom, + hostname, + n_stars, + n_extended, + n_cr, + n_astrom, + path_base, + code + ); + + if (!camProcessedExpInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + psFree(pendingRow); + return false; + } + + // since there is only one exp per 'new' set camRun.state = 'full' + if (!pxcamRunSetState(config, row->cam_id, "full")) { + psError(PS_ERR_UNKNOWN, false, "failed to change camRun.state for cam_id: %" PRId64, row->cam_id); + psFree(row); + psFree(pendingRow); + return false; + } + + // NULL for end_stage means go as far as possible + // EAM : skip here if code != 0 + // Also, we can run fake even if tess_id is not defined + if (code || (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "cam"))) { + psFree(row); + psFree(pendingRow); + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } + psFree(row); + // else continue on... + + if (!pxfakeQueueByCamID(config, + pendingRow->cam_id, + pendingRow->workdir, + pendingRow->label, + pendingRow->reduction, + pendingRow->expgroup, + pendingRow->dvodb, + pendingRow->tess_id, + pendingRow->end_stage + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to queue new fakeRun"); + psFree(pendingRow); + return false; + } + psFree(pendingRow); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + // generate restrictions + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + pxcamGetSearchArgs (config, where); + PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "=="); + + psString query = pxDataGet("camtool_find_processedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND camProcessedExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND camProcessedExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "camProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + pxcamGetSearchArgs (config, where); + PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "camProcessedExp.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = pxDataGet("camtool_reset_faulted_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + + { + psString query = pxDataGet("camtool_revertprocessedexp.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updateprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(code, config->args, "-code", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + if (!pxSetFaultCode(config->dbh, "camProcessedExp", where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + if (!camMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM camMask"); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "camMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + char *query = "DELETE FROM camMask WHERE label = '%s'"; + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("camtool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "camPendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool pendingcleanupexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (cam_id) { + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + } + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("camtool_pendingcleanupexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "camPendingCleanupExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("camtool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "camDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/camtool.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/camtool.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/camtool.h (revision 22161) @@ -0,0 +1,45 @@ +/* + * camtool.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef CAMTOOL_H +#define CAMTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + CAMTOOL_MODE_NONE = 0x0, + CAMTOOL_MODE_DEFINEBYQUERY, + CAMTOOL_MODE_UPDATERUN, + CAMTOOL_MODE_PENDINGEXP, + CAMTOOL_MODE_PENDINGIMFILE, + CAMTOOL_MODE_ADDPROCESSEDEXP, + CAMTOOL_MODE_PROCESSEDEXP, + CAMTOOL_MODE_REVERTPROCESSEDEXP, + CAMTOOL_MODE_UPDATEPROCESSEDEXP, + CAMTOOL_MODE_BLOCK, + CAMTOOL_MODE_MASKED, + CAMTOOL_MODE_UNBLOCK, + CAMTOOL_MODE_PENDINGCLEANUPRUN, + CAMTOOL_MODE_PENDINGCLEANUPEXP, + CAMTOOL_MODE_DONECLEANUP, +} camtoolMode; + +pxConfig *camtoolConfig(pxConfig *config, int argc, char **argv); + +#endif // CAMTOOL_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/camtoolConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/camtoolConfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/camtoolConfig.c (revision 22161) @@ -0,0 +1,231 @@ +/* + * camtoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include + +#include "pxtools.h" +#include "pxcam.h" +#include "camtool.h" + +pxConfig *camtoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *definebyqueryArgs = psMetadataAlloc(); + pxcamSetSearchArgs(definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "search for label", NULL); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_expgroup", 0, "define exposure group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tess ID", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -updaterun + // XXX need to allow multiple cam_ids + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + pxcamSetSearchArgs(updaterunArgs); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search for label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search for state", NULL); + + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + pxcamSetSearchArgs(pendingexpArgs); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", 0, "search for label", NULL); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -pendingimfile + psMetadata *pendingimfileArgs = psMetadataAlloc(); + psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + pxcamSetSearchArgs(pendingimfileArgs); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label", 0, "search for label", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class", 0, "search by class", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addprocessedexp + psMetadata *addprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "define camtool ID (required)", 0); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-uri", 0, "define URI (required)", NULL); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-bias", 0, "define bias", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-bias_stdev", 0, "define bias stdev", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_0", 0, "define fringe term 0", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_1", 0, "define fringe term 1", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_2", 0, "define fringe term 2", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-sigma_ra", 0, "define exposure E ra", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-sigma_dec", 0, "define exposure E dec", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-ap_resid", 0, "define aperture residual", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-ap_resid_stdev", 0, "define aperture residual stdev", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-zp_mean", 0, "define zero point mean", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-zp_stdev", 0, "define zero point stdev", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fwhm_major", 0, "define FWHM (major axis; arcsec)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fwhm_minor", 0, "define FWHM (minor axis; arcsec)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_detrend", 0, "define elapsed detrend processing time (seconds)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_photom", 0, "define elapsed photometry processing time (seconds)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_astrom", 0, "define elapsed astrometry processing time (seconds)", NAN); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_stars", 0, "define number of stars", 0); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_extended", 0, "define number of extended objects", 0); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_cr", 0, "define number of cosmic rays", 0); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_astrom", 0, "define number of astrometry reference objects", 0); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddBool(addprocessedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + + // -processedexp + psMetadata *processedexpArgs = psMetadataAlloc(); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + pxcamSetSearchArgs(processedexpArgs); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label", 0, "search for label", NULL); + + psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + + // -revertprocessedexp + // XXX need to allow multiple cam_ids + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *revertprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + pxcamSetSearchArgs(revertprocessedexpArgs); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-label", 0, "search for label", NULL); + + psMetadataAddBool(revertprocessedexpArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updateprocessedexp + // XXX allow full search options? + psMetadata *updateprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chiptool ID", 0); + psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-class", 0, "search by class", NULL); + psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-code", 0, "set fault code (required)", INT16_MAX); + + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + + // -pendingcleanuprun + // XXX allow full search options? + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupexp + // XXX allow full search options? + psMetadata *pendingcleanupexpArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupexpArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camera ID", 0); + psMetadataAddStr(pendingcleanupexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", NULL); + psMetadataAddBool(pendingcleanupexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from chip stage", CAMTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "change cam run properties", CAMTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingexp", "show pending exposures", CAMTOOL_MODE_PENDINGEXP, pendingexpArgs); + PXOPT_ADD_MODE("-pendingimfile", "show pending imfiles", CAMTOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + PXOPT_ADD_MODE("-addprocessedexp", "add a processed exposure", CAMTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs); + PXOPT_ADD_MODE("-processedexp", "show processed exposures", CAMTOOL_MODE_PROCESSEDEXP, processedexpArgs); + PXOPT_ADD_MODE("-revertprocessedexp", "change procesed exp properties", CAMTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpArgs); + PXOPT_ADD_MODE("-updateprocessedexp", "undo a processed exposure", CAMTOOL_MODE_UPDATEPROCESSEDEXP,updateprocessedexpArgs); + PXOPT_ADD_MODE("-block", "set a label block", CAMTOOL_MODE_BLOCK, blockArgs); + PXOPT_ADD_MODE("-masked", "show blocked labels", CAMTOOL_MODE_MASKED, maskedArgs); + PXOPT_ADD_MODE("-unblock", "remove a label block", CAMTOOL_MODE_UNBLOCK, unblockArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", CAMTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); + PXOPT_ADD_MODE("-pendingcleanupexp", "show exposures for cleanup runs", CAMTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpArgs); + PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", CAMTOOL_MODE_DONECLEANUP, donecleanupArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/chiptool.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/chiptool.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/chiptool.c (revision 22161) @@ -0,0 +1,1149 @@ +/* + * chiptool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "pxdata.h" +#include "pxchip.h" + +#include "chiptool.h" +#include "camtool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool pendingimfileMode(pxConfig *config); +static bool addprocessedimfileMode(pxConfig *config); +static bool processedimfileMode(pxConfig *config); +static bool revertprocessedimfileMode(pxConfig *config); +static bool updateprocessedimfileMode(pxConfig *config); +static bool blockMode(pxConfig *config); +static bool maskedMode(pxConfig *config); +static bool unmaskedMode(pxConfig *config); +static bool unblockMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupimfileMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); +static bool runMode(pxConfig *config); +static bool tocleanedimfileMode(pxConfig *config); +static bool tofullimfileMode(pxConfig *config); +static bool topurgedimfileMode(pxConfig *config); + +static bool chipProcessedCompleteExp(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) { + psLibInit(NULL); + + pxConfig *config = chiptoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(CHIPTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(CHIPTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(CHIPTOOL_MODE_PENDINGIMFILE, pendingimfileMode); + MODECASE(CHIPTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileMode); + MODECASE(CHIPTOOL_MODE_PROCESSEDIMFILE, processedimfileMode); + MODECASE(CHIPTOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileMode); + MODECASE(CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileMode); + MODECASE(CHIPTOOL_MODE_BLOCK, blockMode); + MODECASE(CHIPTOOL_MODE_MASKED, maskedMode); + MODECASE(CHIPTOOL_MODE_UNMASKED, unmaskedMode); + MODECASE(CHIPTOOL_MODE_UNBLOCK, unblockMode); + MODECASE(CHIPTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); + MODECASE(CHIPTOOL_MODE_PENDINGCLEANUPIMFILE, pendingcleanupimfileMode); + MODECASE(CHIPTOOL_MODE_DONECLEANUP, donecleanupMode); + MODECASE(CHIPTOOL_MODE_RUN, runMode); + MODECASE(CHIPTOOL_MODE_TOCLEANEDIMFILE, tocleanedimfileMode); + MODECASE(CHIPTOOL_MODE_TOFULLIMFILE, tofullimfileMode); + MODECASE(CHIPTOOL_MODE_TOPURGEDIMFILE, topurgedimfileMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // rawExp only + PXOPT_COPY_STR(config->args, where, "-label", "rawExp.label", "LIKE"); + + // psListLength(where->list) is at least 1 because exp_type defaults to "object" + // so we require a list longer than 1 entry + if ((psListLength(where->list) <= 1) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + + // default + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("chiptool_find_rawexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + + psFree(whereClause); + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // loop over our list of exp_ids + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + bool status; + psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_id"); + psFree(output); + return false; + } + + // queue the exp + if (!pxchipQueueByExpTag(config, exp_id, workdir, label, reduction, expgroup, dvodb, tess_id, end_stage)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue exp_id: %" PRId64, exp_id); + psFree(output); + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // rawExp, chipRun + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + where = NULL; + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + if ((!state) && (!label)) { + psError(PXTOOLS_ERR_DATA, false, "parameters are required"); + psFree(where); + return false; + } + + if (state) { + // set chipRun.state to state + if (!pxchipRunSetStateByQuery(config, where, state)) { + psFree(where); + return false; + } + } + + if (label) { + // set chipRun.label to label + if (!pxchipRunSetLabelByQuery(config, where, label)) { + psFree(where); + return false; + } + } + + psFree(where); + + return true; +} + + +static bool pendingimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); //chipRun, rawExp + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "=="); + + psString query = pxDataGet("chiptool_pendingimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipPendingImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // chip_id, exp_tag, class_id are required + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + // optional + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_F32(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F32(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F32(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F32(bias, config->args, "-bias", false, false); + PXOPT_LOOKUP_F32(bias_stdev, config->args, "-bias_stdev", false, false); + PXOPT_LOOKUP_F32(fringe_0, config->args, "-fringe_0", false, false); + PXOPT_LOOKUP_F32(fringe_1, config->args, "-fringe_1", false, false); + PXOPT_LOOKUP_F32(fringe_2, config->args, "-fringe_2", false, false); + PXOPT_LOOKUP_F32(sigma_ra, config->args, "-sigma_ra", false, false); + PXOPT_LOOKUP_F32(sigma_dec, config->args, "-sigma_dec", false, false); + PXOPT_LOOKUP_F32(ap_resid, config->args, "-ap_resid", false, false); + PXOPT_LOOKUP_F32(ap_resid_stdev, config->args, "-ap_resid_stdev", false, false); + PXOPT_LOOKUP_F32(zp_mean, config->args, "-zp_mean", false, false); + PXOPT_LOOKUP_F32(zp_stdev, config->args, "-zp_stdev", false, false); + PXOPT_LOOKUP_F32(fwhm_major, config->args, "-fwhm_major", false, false); + PXOPT_LOOKUP_F32(fwhm_minor, config->args, "-fwhm_minor", false, false); + PXOPT_LOOKUP_F32(dtime_detrend, config->args, "-dtime_detrend", false, false); + PXOPT_LOOKUP_F32(dtime_photom, config->args, "-dtime_photom", false, false); + PXOPT_LOOKUP_F32(dtime_astrom, config->args, "-dtime_astrom", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_F32(n_stars, config->args, "-n_stars", false, false); + PXOPT_LOOKUP_F32(n_extended, config->args, "-n_extended", false, false); + PXOPT_LOOKUP_F32(n_cr, config->args, "-n_cr", false, false); + PXOPT_LOOKUP_F32(n_astrom, config->args, "-n_astrom", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!chipProcessedImfileInsert(config->dbh, + chip_id, + exp_id, + class_id, + "full", + uri, + bg, + bg_stdev, + bg_mean_stdev, + bias, + bias_stdev, + fringe_0, + fringe_1, + fringe_2, + sigma_ra, + sigma_dec, + ap_resid, + ap_resid_stdev, + zp_mean, + zp_stdev, + fwhm_major, + fwhm_minor, + dtime_detrend, + dtime_photom, + dtime_astrom, + hostname, + n_stars, + n_extended, + n_cr, + n_astrom, + path_base, + code + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // XXX I've decided to make the transaction cover the Exp migration as + // well. Otherwise, if the last imfile in an exp is moved and the exp + // migration fails then the data base is left in a situation where the exp + // migration can't happen. + + if (!chipProcessedCompleteExp(config)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // chipRun, chipProcessedImfile, rawExp + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "LIKE"); + + psString query = pxDataGet("chiptool_processedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND chipProcessedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipProcessedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool revertprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // chipProcessedImfile, rawExp + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "chipProcessedImfile.fault", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("chiptool_revertprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + +static bool updateprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_LOOKUP_S16(code, config->args, "-code", true, false); + + if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + return false; + } + psFree(where); + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + if (!chipMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (where->list->n < 1) { + psFree(where); + where = NULL; + } + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM chipMask"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psFree(where); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +// return the list of labels which are NOT blocked +static bool unmaskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (where->list->n < 1) { + psFree(where); + where = NULL; + } + + psString query = pxDataGet("chiptool_unmasked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, "chipUnmask"); + psFree(where); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipUnmask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + char *query = "DELETE FROM chipMask WHERE label = '%s'"; + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("chiptool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipPendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool pendingcleanupimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (chip_id) { + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + } + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("chiptool_pendingcleanupimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipPendingCleanupImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("chiptool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool runMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // make sure that the state string is valid + if (!pxIsValidState(state)) { + psError(PXTOOLS_ERR_DATA, false, "%s is not a valid state", state); + return false; + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + + psString query = pxDataGet("chiptool_run.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool chipProcessedCompleteExp(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // look for completed chipPendingExp + // migrate them to chipProccessedExp & camPendingExp + psString query = pxDataGet("chiptool_completely_processed_exp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + chipRunRow *chipRun = chipRunObjectFromMetadata(row); + // set chipRun.state to 'stop' + if (!pxchipRunSetState(config, chipRun->chip_id, "full")) { + psError(PS_ERR_UNKNOWN, false, "failed to change chipRun.state for chip_id: %" PRId64, chipRun->chip_id); + psFree(chipRun); + psFree(output); + return false; + } + + // should we stop here or proceed on to the cam stage? + // NULL for end_stage means go as far as possible + if (chipRun->end_stage && psStrcasestr(chipRun->end_stage, "chip")) { + psFree(chipRun); + continue; + } + // else continue on... + + // camQueueChipID() can only be run after chipRun.state has been set to + // stop + if (!pxcamQueueByChipID(config, + chipRun->chip_id, + chipRun->workdir, + chipRun->label, + chipRun->reduction, + chipRun->expgroup, + chipRun->dvodb, + chipRun->tess_id, + chipRun->end_stage + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to queue camPendingExp"); + psFree(chipRun); + psFree(output); + return false; + } + psFree(chipRun); + } + + psFree(output); + + return true; +} + +// update chipProcessedImfile.data_state to given value. +// afterwards, if all imfiles in the exposure have the new state, update the state for the exposure as well +// shared code for the modes -tocleanedimfile -tofullimfile -topurgedimfile + +static bool change_imfile_data_state(pxConfig *config, psString data_state, psString run_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // chip_id, class_id are required + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + psString query = pxDataGet("chiptool_change_imfile_data_state.sql"); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // note only updates if chipRun.state = run_state + if (!p_psDBRunQuery(config->dbh, query, data_state, chip_id, class_id, run_state)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + // XXX Comment copied from addpendingimfile + // I've decided to make the transaction cover the Exp migration as + // well. Otherwise, if the last imfile in an exp is moved and the exp + // migration fails then the data base is left in a situation where the exp + // migration can't happen. + + query = pxDataGet("chiptool_change_exp_state.sql"); + if (!p_psDBRunQuery(config->dbh, query, data_state, chip_id, data_state)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} +static bool tocleanedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "cleaned", "goto_cleaned"); +} +static bool tofullimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "full", "update"); +} +static bool topurgedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "purged", "goto_purged"); +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/chiptool.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/chiptool.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/chiptool.h (revision 22161) @@ -0,0 +1,50 @@ +/* + * chiptool.h + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef CHIPTOOL_H +#define CHIPTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + CHIPTOOL_MODE_NONE = 0x0, + CHIPTOOL_MODE_DEFINEBYQUERY, + CHIPTOOL_MODE_UPDATERUN, + CHIPTOOL_MODE_PENDINGIMFILE, + CHIPTOOL_MODE_ADDPROCESSEDIMFILE, + CHIPTOOL_MODE_PROCESSEDIMFILE, + CHIPTOOL_MODE_REVERTPROCESSEDIMFILE, + CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE, + CHIPTOOL_MODE_BLOCK, + CHIPTOOL_MODE_MASKED, + CHIPTOOL_MODE_UNMASKED, + CHIPTOOL_MODE_UNBLOCK, + CHIPTOOL_MODE_RETRYPROCESSEDIMFILE, + CHIPTOOL_MODE_PENDINGCLEANUPRUN, + CHIPTOOL_MODE_PENDINGCLEANUPIMFILE, + CHIPTOOL_MODE_DONECLEANUP, + CHIPTOOL_MODE_RUN, + CHIPTOOL_MODE_TOCLEANEDIMFILE, + CHIPTOOL_MODE_TOFULLIMFILE, + CHIPTOOL_MODE_TOPURGEDIMFILE +} chiptoolMode; + +pxConfig *chiptoolConfig(pxConfig *config, int argc, char **argv); + +#endif // CHIPTOOL_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/chiptoolConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/chiptoolConfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/chiptoolConfig.c (revision 22161) @@ -0,0 +1,247 @@ +/* + * chiptoolConfig.c + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "chiptool.h" + +pxConfig *chiptoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + pxchipSetSearchArgs (definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "search by rawExp label (LIKE comparison)", NULL); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-exp_type", PS_META_REPLACE, "search by exp_type", "object"); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_expgroup", 0, "define exposure group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessellation identifier", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + pxchipSetSearchArgs (updaterunArgs); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + + // -pendingimfile + psMetadata *pendingimfileArgs = psMetadataAlloc(); + psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + pxchipSetSearchArgs(pendingimfileArgs); + psMetadataAddU64(pendingimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addprocessedimfile + psMetadata *addprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "define chip ID (required)", 0); + psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID (required)", 0); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID (required)", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-uri", 0, "define URL", NULL); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-bias", 0, "define bias", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-bias_stdev", 0, "define bias stdev", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_0", 0, "define fringe term 0", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_1", 0, "define fringe term 1", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_2", 0, "define fringe term 2", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-sigma_ra", 0, "define error in RA", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-sigma_dec", 0, "define error in DEC", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-ap_resid", 0, "define aperture residual", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-ap_resid_stdev", 0, "define aperture residual stdev", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-zp_mean", 0, "define zero point", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-zp_stdev", 0, "define zero point stdev", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fwhm_major", 0, "define fwhm (major axis; arcsec)", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fwhm_minor", 0, "define fwhm (minor axis; arcsec)", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-dtime_detrend", 0, "define elapsed time for detrend (seconds)", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-dtime_photom", 0, "define elapsed time for photometry (seconds)", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-dtime_astrom", 0, "define elapsed time for astrometry (seconds)", NAN); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL); + psMetadataAddS32(addprocessedimfileArgs, PS_LIST_TAIL, "-n_stars", 0, "define number of stars in image", 0); + psMetadataAddS32(addprocessedimfileArgs, PS_LIST_TAIL, "-n_extended", 0, "define number of extended objects in image", 0); + psMetadataAddS32(addprocessedimfileArgs, PS_LIST_TAIL, "-n_cr", 0, "define number of cosmic rays in image", 0); + psMetadataAddS32(addprocessedimfileArgs, PS_LIST_TAIL, "-n_astrom", 0, "define number of astrometry stars used", 0); + + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -processedimfile + psMetadata *processedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "define chip ID", 0); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-label", 0, "search by chipRun label (LIKE comparison)", NULL); + pxchipSetSearchArgs(processedimfileArgs); + psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertprocessedimfile + psMetadata *revertprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-label", 0, "search by chipRun label (LIKE comparison)", NULL); + pxchipSetSearchArgs(revertprocessedimfileArgs); + psMetadataAddBool(revertprocessedimfileArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddS16(revertprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updateprocessedimfile + psMetadata *updateprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code (required)", 0); + + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddStr(maskedArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(maskedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -unmasked + psMetadata *unmaskedArgs = psMetadataAlloc(); + psMetadataAddStr(unmaskedArgs, PS_LIST_TAIL, "-label", 0, "restrict to specified label", NULL); + psMetadataAddBool(unmaskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(unmaskedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + + // -pendingcleanuprun + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupimfile + psMetadata *pendingcleanupimfileArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", NULL); + psMetadataAddBool(pendingcleanupimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -run + psMetadata *runArgs = psMetadataAlloc(); + psMetadataAddStr(runArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddBool(runArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(runArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddStr(runArgs, PS_LIST_TAIL, "-state", 0, "search by state (required)", NULL); + + // -tocleanedimfile + psMetadata *tocleanedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "chip ID to update", 0); + psMetadataAddStr(tocleanedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + // -tofullimfile + psMetadata *tofullimfileArgs = psMetadataAlloc(); + psMetadataAddS64(tofullimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "chip ID to update", 0); + psMetadataAddStr(tofullimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + // -topurgedimfile + psMetadata *topurgedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(topurgedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "chip ID to update", 0); + psMetadataAddStr(topurgedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from raw exposures", CHIPTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "change chip run properties", CHIPTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingimfile", "show pending imfiles", CHIPTOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + PXOPT_ADD_MODE("-addprocessedimfile", "add a processed imfile", CHIPTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); + PXOPT_ADD_MODE("-processedimfile", "show processed imfiles", CHIPTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); + PXOPT_ADD_MODE("-updateprocessedimfile","change procesed imfile properties", CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE,updateprocessedimfileArgs); + PXOPT_ADD_MODE("-revertprocessedimfile","undo a processed imfile", CHIPTOOL_MODE_REVERTPROCESSEDIMFILE,revertprocessedimfileArgs); + PXOPT_ADD_MODE("-block", "set a label block", CHIPTOOL_MODE_BLOCK, blockArgs); + PXOPT_ADD_MODE("-masked", "show blocked labels", CHIPTOOL_MODE_MASKED, maskedArgs); + PXOPT_ADD_MODE("-unmasked", "", CHIPTOOL_MODE_UNMASKED, unmaskedArgs); + PXOPT_ADD_MODE("-unblock", "remove a label block", CHIPTOOL_MODE_UNBLOCK, unblockArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", CHIPTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); + PXOPT_ADD_MODE("-pendingcleanupimfile", "show runs that need to be cleaned up", CHIPTOOL_MODE_PENDINGCLEANUPIMFILE, pendingcleanupimfileArgs); + PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", CHIPTOOL_MODE_DONECLEANUP, donecleanupArgs); + PXOPT_ADD_MODE("-run", "show runs", CHIPTOOL_MODE_RUN, runArgs); + PXOPT_ADD_MODE("-tocleanedimfile", "set imfile state to cleaned", CHIPTOOL_MODE_TOCLEANEDIMFILE, tocleanedimfileArgs); + PXOPT_ADD_MODE("-tofullimfile", "set imfile state to full", CHIPTOOL_MODE_TOFULLIMFILE, tofullimfileArgs); + PXOPT_ADD_MODE("-topurgdimfile", "set imfile state to purged", CHIPTOOL_MODE_TOPURGEDIMFILE, topurgedimfileArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/detselect.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/detselect.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/detselect.c (revision 22161) @@ -0,0 +1,220 @@ +/* + * detselect.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "detselect.h" + +static bool searchMode(pxConfig *config); +static bool selectMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = detselectConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(DETSELECT_MODE_SEARCH, searchMode); + MODECASE(DETSELECT_MODE_SELECT, selectMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool searchMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(unlimit, config->args, "-unlimit", false); + + PXOPT_LOOKUP_TIME(time, config->args, "-time", false, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-det_type", "det_type", "=="); + PXOPT_COPY_STR(config->args, where, "-type", "det_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + + // airmass_min < airmass < airmass_max + PXOPT_COPY_F32(config->args, where, "-airmass", "airmass_min", "<="); + PXOPT_COPY_F32(config->args, where, "-airmass", "airmass_max", ">="); + + // exp_time_min < exp_time < exp_time_max + PXOPT_COPY_F32(config->args, where, "-exp_time", "exp_time_min", "<="); + PXOPT_COPY_F32(config->args, where, "-exp_time", "exp_time_max", ">="); + + // ccd_temp_min < ccd_temp < ccd_temp_max + PXOPT_COPY_F32(config->args, where, "-ccd_temp", "ccd_temp_min", "<="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp", "ccd_temp_max", ">="); + + PXOPT_COPY_F64(config->args, where, "-posang", "posang_min", "<="); + PXOPT_COPY_F64(config->args, where, "-posang", "posang_max", ">="); + + // time_begin < time < time_end + // the == NULL tests invokes some psDB magic to make an OR + // conditional query + psMetadataAddTime(where, PS_LIST_TAIL, "time_begin", 0, "<=", time); + psMetadataAddTime(where, PS_LIST_TAIL, "time_begin", PS_META_DUPLICATE_OK, "==", NULL); + psMetadataAddTime(where, PS_LIST_TAIL, "time_end", 0, ">=", time); + psMetadataAddTime(where, PS_LIST_TAIL, "time_end", PS_META_DUPLICATE_OK, "==", NULL); + + psString query = pxDataGet("detselect_search.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // we choose the single detrend image which matches all criteria and has + // the latest insertion date + + // unless explicitly specified by the user, list all possible matches + if (!unlimit) { + psStringAppend(&query, " ORDER BY registered DESC LIMIT 1"); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no detrend exposures found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool selectMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString query = pxDataGet("detselect_select.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no detNormalizedImfile rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detNormalizedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/detselect.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/detselect.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/detselect.h (revision 22161) @@ -0,0 +1,33 @@ +/* + * detselect.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef DETSELECT_H +#define DETSELECT_H 1 + +#include "pxtools.h" + +typedef enum { + DETSELECT_MODE_NONE = 0x0, + DETSELECT_MODE_SEARCH, + DETSELECT_MODE_SELECT, +} detselectMode; + +pxConfig *detselectConfig(pxConfig *config, int argc, char **argv); + +#endif // DETSELECT_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/detselectConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/detselectConfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/detselectConfig.c (revision 22161) @@ -0,0 +1,94 @@ +/* + * detselectConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "pxtools.h" +#include "detselect.h" + +pxConfig *detselectConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -search + psMetadata *searchArgs = psMetadataAlloc(); + psMetadataAddStr(searchArgs, PS_LIST_TAIL, "-inst", 0, "search by camera", NULL); + psMetadataAddStr(searchArgs, PS_LIST_TAIL, "-telescope", 0, "search by telescope", NULL); + psMetadataAddStr(searchArgs, PS_LIST_TAIL, "-det_type", 0, "search by detrend type", NULL); + psMetadataAddStr(searchArgs, PS_LIST_TAIL, "-type", 0, "search by detrend type (alias for -det_type)", NULL); + psMetadataAddStr(searchArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddF32(searchArgs, PS_LIST_TAIL, "-airmass", 0, "define airmass", NAN); + psMetadataAddF32(searchArgs, PS_LIST_TAIL, "-exp_time", 0, "search by exposure time", NAN); + psMetadataAddF32(searchArgs, PS_LIST_TAIL, "-ccd_temp", 0, "search by ccd tempature", NAN); + psMetadataAddF64(searchArgs, PS_LIST_TAIL, "-posang", 0, "search by rotator position angle", NAN); + psMetadataAddTime(searchArgs, PS_LIST_TAIL, "-time", 0, "define time for desired detrend data", NULL); + psMetadataAddBool(searchArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(searchArgs, PS_LIST_TAIL, "-unlimit", 0, "list all possible detruns, not just the best match", false); + + // -select + psMetadata *selectArgs = psMetadataAlloc(); + psMetadataAddStr(selectArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID", NULL); + psMetadataAddS32(selectArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(selectArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddBool(selectArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-search", "search for an appropriate detrend", DETSELECT_MODE_SEARCH, searchArgs); + PXOPT_ADD_MODE("-select", "retreive detrend information", DETSELECT_MODE_SELECT, selectArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool.c (revision 22161) @@ -0,0 +1,2057 @@ +/* + * dettool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +static bool pendingMode(pxConfig *config); +static bool definebytagMode(pxConfig *config); +static bool definebyqueryMode(pxConfig *config); +static bool definebydetrunMode(pxConfig *config); +static bool runsMode(pxConfig *config); +static bool childlessrunMode(pxConfig *config); +static bool inputMode(pxConfig *config); +static bool rawMode(pxConfig *config); + +// run +static bool updatedetrunMode(pxConfig *config); +static bool rerunMode(pxConfig *config); +// register +static bool register_detrendMode(pxConfig *config); + +//static psArray *validDetInputClassIds(pxConfig *config, const char *det_id); +//static psArray *searchInputImfiles(pxConfig *config, const char *det_id); +static detInputExpRow *rawDetrenTodetInputExpRow(rawExpRow *rawExp, psS64 det_id, psS32 iteration); +static psS32 incrementIteration(pxConfig *config, psS64 det_id); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = dettoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(DETTOOL_MODE_PENDING, pendingMode); + MODECASE(DETTOOL_MODE_DEFINEBYTAG, definebytagMode); + MODECASE(DETTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(DETTOOL_MODE_DEFINEBYDETRUN, definebydetrunMode); + MODECASE(DETTOOL_MODE_RUNS, runsMode); + MODECASE(DETTOOL_MODE_CHILDLESSRUN, childlessrunMode); + MODECASE(DETTOOL_MODE_INPUT, inputMode); + MODECASE(DETTOOL_MODE_RAW, rawMode); + // correction + MODECASE(DETTOOL_MODE_MAKECORRECTION, makecorrectionMode); + MODECASE(DETTOOL_MODE_TOCORRECTEXP, tocorrectexpMode); + MODECASE(DETTOOL_MODE_TOCORRECTIMFILE, tocorrectimfileMode); + MODECASE(DETTOOL_MODE_ADDCORRECTIMFILE, addcorrectimfileMode); + // imfile + MODECASE(DETTOOL_MODE_TOPROCESSEDIMFILE,toprocessedimfileMode); + MODECASE(DETTOOL_MODE_ADDPROCESSEDIMFILE,addprocessedimfileMode); + MODECASE(DETTOOL_MODE_PROCESSEDIMFILE, processedimfileMode); + MODECASE(DETTOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileMode); + MODECASE(DETTOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDIMFILE, pendingcleanup_processedimfileMode); + MODECASE(DETTOOL_MODE_DONECLEANUP_PROCESSEDIMFILE, donecleanup_processedimfileMode); + // exp + MODECASE(DETTOOL_MODE_TOPROCESSEDEXP, toprocessedexpMode); + MODECASE(DETTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpMode); + MODECASE(DETTOOL_MODE_PROCESSEDEXP, processedexpMode); + MODECASE(DETTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); + MODECASE(DETTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDEXP, pendingcleanup_processedexpMode); + MODECASE(DETTOOL_MODE_DONECLEANUP_PROCESSEDEXP, donecleanup_processedexpMode); + // stacked + MODECASE(DETTOOL_MODE_TOSTACKED, tostackedMode); + MODECASE(DETTOOL_MODE_ADDSTACKED, addstackedMode); + MODECASE(DETTOOL_MODE_STACKED, stackedMode); + MODECASE(DETTOOL_MODE_REVERTSTACKED, revertstackedMode); + MODECASE(DETTOOL_MODE_UPDATESTACKED, updatestackedMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_STACKED, pendingcleanup_stackedMode); + MODECASE(DETTOOL_MODE_DONECLEANUP_STACKED, donecleanup_stackedMode); + // normalizedstat + MODECASE(DETTOOL_MODE_TONORMALIZEDSTAT, tonormalizedstatMode); + MODECASE(DETTOOL_MODE_ADDNORMALIZEDSTAT,addnormalizedstatMode); + MODECASE(DETTOOL_MODE_NORMALIZEDSTAT, normalizedstatMode); + MODECASE(DETTOOL_MODE_REVERTNORMALIZEDSTAT, revertnormalizedstatMode); + MODECASE(DETTOOL_MODE_UPDATENORMALIZEDSTAT, updatenormalizedstatMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_NORMALIZEDSTAT, pendingcleanup_normalizedstatMode); + MODECASE(DETTOOL_MODE_DONECLEANUP_NORMALIZEDSTAT, donecleanup_normalizedstatMode); + // normalizedimfile + MODECASE(DETTOOL_MODE_TONORMALIZE, tonormalizeMode); + MODECASE(DETTOOL_MODE_ADDNORMALIZEDIMFILE,addnormalizedimfileMode); + MODECASE(DETTOOL_MODE_NORMALIZEDIMFILE, normalizedimfileMode); + MODECASE(DETTOOL_MODE_REVERTNORMALIZEDIMFILE, revertnormalizedimfileMode); + MODECASE(DETTOOL_MODE_UPDATENORMALIZEDIMFILE, updatenormalizedimfileMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_NORMALIZEDIMFILE, pendingcleanup_normalizedimfileMode); + MODECASE(DETTOOL_MODE_DONECLEANUP_NORMALIZEDIMFILE, donecleanup_normalizedimfileMode); + // normalizedexp + MODECASE(DETTOOL_MODE_TONORMALIZEDEXP, tonormalizedexpMode); + MODECASE(DETTOOL_MODE_ADDNORMALIZEDEXP, addnormalizedexpMode); + MODECASE(DETTOOL_MODE_NORMALIZEDEXP, normalizedexpMode); + MODECASE(DETTOOL_MODE_REVERTNORMALIZEDEXP, revertnormalizedexpMode); + MODECASE(DETTOOL_MODE_UPDATENORMALIZEDEXP, updatenormalizedexpMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_NORMALIZEDEXP, pendingcleanup_normalizedexpMode); + MODECASE(DETTOOL_MODE_DONECLEANUP_NORMALIZEDEXP, donecleanup_normalizedexpMode); + // residimfile + MODECASE(DETTOOL_MODE_TORESIDIMFILE, toresidimfileMode); + MODECASE(DETTOOL_MODE_ADDRESIDIMFILE, addresidimfileMode); + MODECASE(DETTOOL_MODE_RESIDIMFILE, residimfileMode); + MODECASE(DETTOOL_MODE_REVERTRESIDIMFILE,revertresidimfileMode); + MODECASE(DETTOOL_MODE_UPDATERESIDIMFILE, updateresidimfileMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_RESIDIMFILE, pendingcleanup_residimfileMode); + MODECASE(DETTOOL_MODE_DONECLEANUP_RESIDIMFILE, donecleanup_residimfileMode); + // residexp + MODECASE(DETTOOL_MODE_TORESIDEXP, toresidexpMode); + MODECASE(DETTOOL_MODE_ADDRESIDEXP, addresidexpMode); + MODECASE(DETTOOL_MODE_RESIDEXP, residexpMode); + MODECASE(DETTOOL_MODE_REVERTRESIDEXP, revertresidexpMode); + MODECASE(DETTOOL_MODE_UPDATERESIDEXP, updateresidexpMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_RESIDEXP, pendingcleanup_residexpMode); + MODECASE(DETTOOL_MODE_DONECLEANUP_RESIDEXP, donecleanup_residexpMode); + // detrunsummary + MODECASE(DETTOOL_MODE_TODETRUNSUMMARY, todetrunsummaryMode); + MODECASE(DETTOOL_MODE_ADDDETRUNSUMMARY, adddetrunsummaryMode); + MODECASE(DETTOOL_MODE_DETRUNSUMMARY, detrunsummaryMode); + MODECASE(DETTOOL_MODE_REVERTDETRUNSUMMARY, revertdetrunsummaryMode); + MODECASE(DETTOOL_MODE_UPDATEDETRUNSUMMARY, updatedetrunsummaryMode); + MODECASE(DETTOOL_MODE_UPDATEDETRUN, updatedetrunMode); + MODECASE(DETTOOL_MODE_RERUN, rerunMode); + // register + MODECASE(DETTOOL_MODE_REGISTER_DETREND, register_detrendMode); + MODECASE(DETTOOL_MODE_REGISTER_DETREND_IMFILE, register_detrend_imfileMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool pendingMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_pending.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool definebytagMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // what type of detRun is this? + // required + PXOPT_LOOKUP_STR(det_type, config->args, "-det_type", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + + // optional + PXOPT_LOOKUP_STR(filelevel, config->args, "-filelevel", false, false); + + // check mode + PXOPT_LOOKUP_STR(mode, config->args, "-mode", true, false); // default ('master') is supplied + if (!isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalid mode"); + return false; + } + + // get -ref_det_id and -ref_iter : required for 'verify' mode / disallowed otherwise + PXOPT_LOOKUP_S64(ref_det_id, config->args, "-ref_det_id", true, false); + PXOPT_LOOKUP_S32(ref_iter, config->args, "-ref_iter", true, false); + if (!strncmp(mode, "verify", 7) && ((ref_det_id == 0) || (ref_iter == -1))) { + psError(PS_ERR_UNKNOWN, false, "verify mode requires both -ref_det_id and -ref_iter"); + return false; + } + if (strncmp(mode, "verify", 7) && ((ref_det_id != 0) || (ref_iter != -1))) { + psError(PS_ERR_UNKNOWN, false, "master mode cannot have -ref_det_id or -ref_iter set"); + return false; + } + + PXOPT_LOOKUP_STR(camera, config->args, "-inst", false, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", false, false); + PXOPT_LOOKUP_STR(exp_type, config->args, "-exp_type", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-filter", false, false); + + PXOPT_LOOKUP_F32(airmass_min, config->args, "-airmass_min", false, false); + PXOPT_LOOKUP_F32(airmass_max, config->args, "-airmass_max", false, false); + PXOPT_LOOKUP_F32(exp_time_min, config->args, "-exp_time_min", false, false); + PXOPT_LOOKUP_F32(exp_time_max, config->args, "-exp_time_max", false, false); + PXOPT_LOOKUP_F32(ccd_temp_min, config->args, "-ccd_temp_min", false, false); + PXOPT_LOOKUP_F32(ccd_temp_max, config->args, "-ccd_temp_max", false, false); + PXOPT_LOOKUP_F64(posang_min, config->args, "-posang_min", false, false); + PXOPT_LOOKUP_F64(posang_max, config->args, "-posang_max", false, false); + PXOPT_LOOKUP_F64(solang_min, config->args, "-solang_min", false, false); + PXOPT_LOOKUP_F64(solang_max, config->args, "-solang_max", false, false); + PXOPT_LOOKUP_TIME(time_begin, config->args, "-time_begin", false, false); + PXOPT_LOOKUP_TIME(time_end, config->args, "-time_end", false, false); + PXOPT_LOOKUP_TIME(use_begin, config->args, "-use_begin", false, false); + PXOPT_LOOKUP_TIME(use_end, config->args, "-use_end", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + + // default + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + + // we have to support multipe exp_ids + psMetadataItem *item = psMetadataLookup(config->args, "-exp_id"); + if (!item) { + // this shouldn't actually happen when using psArgs + psError(PS_ERR_UNKNOWN, true, "-exp_id is required"); + return false; + } + psMetadata *where = psMetadataAlloc(); + + // make sure that -exp_id was parsed correctly + // XXX this can be removed someday + // XXX Special case for a multi : move into a macro? + if (item->type == PS_DATA_METADATA_MULTI) { + psListIterator *iter = psListIteratorAlloc(item->data.list, 0, false); + psMetadataItem *mItem = NULL; + while ((mItem = psListGetAndIncrement(iter))) { + psString exp_id = mItem->data.V; + // if exp_id is NULL then it means that -exp_id has not been + // specified + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, + "at least one -exp_id is required"); + psFree(where); + return false; + } + + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", + PS_META_DUPLICATE_OK, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(iter); + psFree(where); + return false; + } + } + psFree(iter); + } else { + psAbort( "-exp_id was not parsed correctly (this should not happen"); + } + + if (psListLength(where->list) < 1) { + psFree(where); + where = NULL; + } + + // check that the specified exp_ids actually exist + psArray *detrendExps = rawExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + if (!detrendExps) { + psError(PS_ERR_UNKNOWN, false, "no rawExp rows found"); + return false; + } + + // we should have one rawExp row per exp_id specified + if (psListLength(item->data.list) != psArrayLength(detrendExps)) { + psAbort( "an -exp_id matched more then one rawExp (this should not happen"); + + } + + // check to see if -filelevel was set on the command line + if (!filelevel) { + filelevel = psStringCopy(((rawExpRow *)detrendExps->data[0])->filelevel); + } + + // start a transaction so we don't end up with childlessed det_ids + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detrendExps); + return false; + } + + // the first iteration is always 0 + // XXX the camera name is set from the first inputExp + // XXX det_id + detRunInsert(config->dbh, + 0, + 0, + det_type, + mode, + "run", + filelevel, + workdir, + camera, + telescope, + exp_type, + reduction, + filter, + airmass_min, + airmass_max, + exp_time_min, + exp_time_max, + ccd_temp_min, + ccd_temp_max, + posang_min, + posang_max, + registered, + time_begin, + time_end, + use_begin, + use_end, + solang_min, + solang_max, + label, + ref_det_id, + ref_iter + ); + psS64 det_id = psDBLastInsertID(config->dbh); + + // create new detInputExp row(s) from the rawExp row(s) + psArray *inputExps = psArrayAllocEmpty(psArrayLength(detrendExps)); + for (long i = 0; i < psArrayLength(detrendExps); i++) { + detInputExpRow *inputExp = rawDetrenTodetInputExpRow( + detrendExps->data[i], + det_id, + 0 // the first iteration is explicitly 0 + ); + psArrayAdd(inputExps, 0, inputExp); + psFree(inputExp); + } + + psFree(detrendExps); + + // insert detInputExp objects into the database + if (!detInputExpInsertObjects(config->dbh, inputExps)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(inputExps); + return false; + } + psFree(inputExps); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // print the new det_id + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id); + detRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check results + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 " (this should not happen)", det_id); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, detRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detRuns); + return false; + } + psFree(detRuns); + + return true; +} + +#if 0 +// This function is used to convert the det_id from an int, as it is used +// internally, to be a string for external use. The rationale being that we may +// want to change how det_id is generated in the future and don't want to +// external programs to become depending on this value being an int. +static bool convertDetIdToStr(psArray *mds) +{ + PS_ASSERT_PTR_NON_NULL(mds, false); + + for (long i = 0; i < psArrayLength(mds); i++) { + psMetadata *md = mds->data[i]; + bool status = false; + psS32 det_id = psMetadataLookupS32(&status, md, "det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for det_id"); + return false; + } + psMetadataRemoveKey(md, "det_id"); + psString det_idStr = psDBIntToString((psU64)det_id); + psMetadataAddStr(mds->data[i], PS_LIST_HEAD, "det_id", 0, NULL, det_idStr); + psFree(det_idStr); + } + + return true; +} +#endif + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(det_type, config->args, "-det_type", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + + // check mode + PXOPT_LOOKUP_STR(mode, config->args, "-mode", true, false); // default ('master') is supplied + if (!isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalid mode"); + return false; + } + + // get -ref_det_id and -ref_iter : required for 'verify' mode / disallowed otherwise + PXOPT_LOOKUP_S64(ref_det_id, config->args, "-ref_det_id", false, false); + PXOPT_LOOKUP_S32(ref_iter, config->args, "-ref_iter", false, false); + if (!strncmp(mode, "verify", 7) && ((ref_det_id == 0) || (ref_iter == -1))) { + psError(PS_ERR_UNKNOWN, false, "verify mode requires both -ref_det_id and -ref_iter"); + return false; + } + if (strncmp(mode, "verify", 7) && ((ref_det_id != 0) || (ref_iter != -1))) { + psError(PS_ERR_UNKNOWN, false, "master mode cannot have -ref_det_id or -ref_iter set"); + return false; + } + + PXOPT_LOOKUP_STR(filelevel, config->args, "-filelevel", false, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-filter", false, false); + PXOPT_LOOKUP_F32(airmass_min, config->args, "-airmass_min", false, false); + PXOPT_LOOKUP_F32(airmass_max, config->args, "-airmass_max", false, false); + PXOPT_LOOKUP_F32(exp_time_min, config->args, "-exp_time_min", false, false); + PXOPT_LOOKUP_F32(exp_time_max, config->args, "-exp_time_max", false, false); + PXOPT_LOOKUP_F32(ccd_temp_min, config->args, "-ccd_temp_min", false, false); + PXOPT_LOOKUP_F32(ccd_temp_max, config->args, "-ccd_temp_max", false, false); + PXOPT_LOOKUP_F64(posang_min, config->args, "-posang_min", false, false); + PXOPT_LOOKUP_F64(posang_max, config->args, "-posang_max", false, false); + PXOPT_LOOKUP_F64(solang_min, config->args, "-solang_min", false, false); + PXOPT_LOOKUP_F64(solang_max, config->args, "-solang_max", false, false); + + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + PXOPT_LOOKUP_BOOL(randomSubset, config->args, "-random_subset", false); + PXOPT_LOOKUP_S32(randomLimit, config->args, "-random_limit", false, false); + + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_TIME(time_begin, config->args, "-time_begin", false, false); + PXOPT_LOOKUP_TIME(time_end, config->args, "-time_end", false, false); + PXOPT_LOOKUP_TIME(use_begin, config->args, "-use_begin", false, false); + PXOPT_LOOKUP_TIME(use_end, config->args, "-use_end", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-select_exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-select_inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-select_telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-select_filter", "filter", "=="); + PXOPT_COPY_STR(config->args, where, "-select_uri", "uri", "=="); + PXOPT_COPY_TIME(config->args, where, "-select_dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-select_dateobs_end", "dateobs", "<="); + PXOPT_COPY_F32(config->args, where, "-select_airmass_min", "airmass", ">="); + PXOPT_COPY_F32(config->args, where, "-select_airmass_max", "airmass", "<="); + PXOPT_COPY_F32(config->args, where, "-select_sat_pixel_frac_max", "sat_pixel_frac", "<="); + PXOPT_COPY_F32(config->args, where, "-select_exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-select_exp_time_max", "exp_time", "<="); + PXOPT_COPY_F64(config->args, where, "-select_ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F64(config->args, where, "-select_ccd_temp_max", "ccd_temp", "<="); + PXOPT_COPY_F64(config->args, where, "-select_pon_time_min", "pon_time", ">="); + PXOPT_COPY_F64(config->args, where, "-select_pon_time_max", "pon_time", "<="); + PXOPT_COPY_F64(config->args, where, "-select_posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-select_posang_max", "posang", "<="); + PXOPT_COPY_F64(config->args, where, "-select_solang_min", "solang", ">="); + PXOPT_COPY_F64(config->args, where, "-select_solang_max", "solang", "<="); + + if (!psListLength(where->list)) { + psFree(where); + where = NULL; + } + + // there is some namespace overlap between the names of the fields we'd + // like to search by to setup a detrun and the names of the fields we'd + // like to assign values to so I've separated them but prepending set- to + // the assigned values + + // search for rawExps with the specified options + psArray *detrendExps = rawExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + // make sure that we found at least one rawExp + if (!detrendExps) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(detrendExps)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(detrendExps); + return true; + } + + if (randomSubset && (randomLimit < detrendExps->n)) { + // generate a random-valued vector, return an index sorted by the random values + psVector *randomVector = psVectorAlloc(detrendExps->n, PS_TYPE_F32); // random values + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); + for (int i = 0; i < randomVector->n; i++) { + randomVector->data.F32[i] = psRandomUniform(rng); + } + psVector *indexVector = psVectorSortIndex(NULL, randomVector); + + // accept for first n of the sequence + psArray *subset = psArrayAlloc (randomLimit); + for (int i = 0; i < randomLimit; i++ ){ + int j = indexVector->data.S32[i]; + subset->data[i] = psMemIncrRefCounter (detrendExps->data[j]); + } + psFree (detrendExps); + detrendExps = subset; + } + + // check to see if -filelevel was set on the command line + if (!filelevel) { + filelevel = psStringCopy(((rawExpRow *)detrendExps->data[0])->filelevel); + } + + if (pretend) { + // negative simple so the default is true + if (!rawExpPrintObjects(stdout, detrendExps, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detrendExps); + return false; + } + psFree(detrendExps); + return true; + } + + // start a transaction so we don't end up with childlessed det_ids + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detrendExps); + return false; + } + + // the first iteration is always 0 + // XXX det_id + detRunInsert(config->dbh, + 0, // det_id + 0, // iteration + det_type, + mode, + "run", // state + filelevel, + workdir, + camera, + telescope, + "NA", + reduction, + filter, + airmass_min, + airmass_max, + exp_time_min, + exp_time_max, + ccd_temp_min, + ccd_temp_max, + posang_min, + posang_max, + registered, + time_begin, + time_end, + use_begin, + use_end, + solang_min, + solang_max, + label, + ref_det_id, + ref_iter + ); + psS64 det_id = psDBLastInsertID(config->dbh); + + // create new detInputExp row(s) from the rawExp row(s) + psArray *inputExps = psArrayAllocEmpty(psArrayLength(detrendExps)); + for (long i = 0; i < psArrayLength(detrendExps); i++) { + detInputExpRow *inputExp = rawDetrenTodetInputExpRow( + detrendExps->data[i], + det_id, + 0 // the first iteration is explicitly 0 + ); + psArrayAdd(inputExps, 0, inputExp); + psFree(inputExp); + } + + psFree(detrendExps); + + // insert detInputExp objects into the database + if (!detInputExpInsertObjects(config->dbh, inputExps)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(inputExps); + return false; + } + psFree(inputExps); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + // print the new det_id + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id); + detRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check results + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 " (this should not happen)", det_id); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, detRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detRuns); + return false; + } + psFree(detRuns); + + return true; +} + +static bool definebydetrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // XXX this mode is not well-tested: probably need to specify iteration here + // XXX pass the supplied det_id and iteration as ref_det_id, or get from command line? + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_STR(det_type, config->args, "-set_det_type", false, false); // optional + PXOPT_LOOKUP_STR(mode, config->args, "-set_mode", false, false); // optional + PXOPT_LOOKUP_STR(exp_type, config->args, "-set_exp_type", false, false); + PXOPT_LOOKUP_STR(filelevel, config->args, "-set_filelevel", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-set_filter", false, false); + PXOPT_LOOKUP_F32(airmass_min, config->args, "-set_airmass_min", false, false); + PXOPT_LOOKUP_F32(airmass_max, config->args, "-set_airmass_max", false, false); + PXOPT_LOOKUP_F32(exp_time_min, config->args, "-set_exp_time_min", false, false); + PXOPT_LOOKUP_F32(exp_time_max, config->args, "-set_exp_time_max", false, false); + PXOPT_LOOKUP_F64(ccd_temp_min, config->args, "-set_ccd_temp_min", false, false); + PXOPT_LOOKUP_F64(ccd_temp_max, config->args, "-set_ccd_temp_max", false, false); + PXOPT_LOOKUP_F64(posang_min, config->args, "-set_posang_min", false, false); + PXOPT_LOOKUP_F64(posang_max, config->args, "-set_posang_max", false, false); + PXOPT_LOOKUP_F64(solang_min, config->args, "-set_solang_min", false, false); + PXOPT_LOOKUP_F64(solang_max, config->args, "-set_solang_max", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + PXOPT_LOOKUP_TIME(time_begin, config->args, "-set_time_begin", false, false); + PXOPT_LOOKUP_TIME(time_end, config->args, "-set_end", false, false); + PXOPT_LOOKUP_TIME(use_begin, config->args, "-use_begin", false, false); + PXOPT_LOOKUP_TIME(use_end, config->args, "-use_end", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); // optional + + // check mode + if (mode && !isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalud mode"); + return false; + } + + // lookup the detRun that we will be basing this one on + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id); + detRuns = detRunSelectRowObjects(config->dbh, where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + // sanity check the result... we should have only found one det_id + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 " (this should not happen)", det_id); + return false; // unreachable + } + + // pull the detRun object out the result array + // XXX need to assign or re-assign the ref_det_id and ref_iter values + detRunRow *detRun = psMemIncrRefCounter(detRuns->data[0]); + + // discard the resultarray + psFree(detRuns); + + // set the det_id to 0/NULL so the database can assign it + detRun->det_id = 0; + + // reset the iteration to 0 + detRun->iteration = 0; + + // reset the state to "run" + psFree(detRun->state); + detRun->state = psStringCopy("run"); + + // walk through the optional values and update the detRun as required + if (det_type) { + psFree(detRun->det_type); + detRun->det_type = psStringCopy(det_type); + } + + if (mode) { + psFree(detRun->mode); + detRun->mode = psStringCopy(mode); + } + + if (exp_type) { + psFree(detRun->exp_type); + detRun->exp_type = psStringCopy(exp_type); + } + + if (filelevel) { + psFree(detRun->filelevel); + detRun->filelevel = psStringCopy(filelevel); + } + + if (workdir) { + psFree(detRun->workdir); + detRun->workdir = psStringCopy(workdir); + } + + if (filter) { + psFree(detRun->filter); + detRun->filter = psStringCopy(filter); + } + + if (!isnan(airmass_min)) { + detRun->airmass_min = airmass_min; + } + + if (!isnan(airmass_max)) { + detRun->airmass_max = airmass_max; + } + + if (!isnan(exp_time_min)) { + detRun->exp_time_min = exp_time_min; + } + + if (!isnan(exp_time_max)) { + detRun->exp_time_max = exp_time_max; + } + + if (!isnan(ccd_temp_min)) { + detRun->ccd_temp_min = ccd_temp_min; + } + + if (!isnan(ccd_temp_max)) { + detRun->ccd_temp_max = ccd_temp_max; + } + + if (!isnan(posang_min)) { + detRun->posang_min = posang_min; + } + + if (!isnan(posang_max)) { + detRun->posang_max = posang_max; + } + + if (!isnan(solang_min)) { + detRun->solang_min = solang_min; + } + + if (!isnan(solang_max)) { + detRun->solang_max = solang_max; + } + + if (label) { + psFree(detRun->label); + detRun->label = label; + } + + if (registered) { + psFree(detRun->registered); + detRun->registered = psMemIncrRefCounter(registered); + } + + if (time_begin) { + psFree(detRun->time_begin); + detRun->time_begin = psMemIncrRefCounter(time_begin); + } + + if (time_end) { + psFree(detRun->time_end); + detRun->time_end = psMemIncrRefCounter(time_end); + } + + if (use_begin) { + psFree(detRun->use_begin); + detRun->use_begin = psMemIncrRefCounter(use_begin); + } + + if (use_end) { + psFree(detRun->use_end); + detRun->use_end = psMemIncrRefCounter(use_end); + } + + if (reduction) { + psFree(detRun->reduction); + detRun->reduction = psStringCopy(reduction); + } + + // create a metadata to restrict detInputExp's be in in the specified range + + psMetadata *time_filter = psMetadataAlloc(); + + // XXX: possible mem leak: PXOPT_COPY* will free time_filter but NOTHING + // that has previously been allocated + PXOPT_COPY_TIME(config->args, time_filter, "-set_input_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, time_filter, "-set_input_end", "dateobs", "<"); + + // start a transaction so we don't end up with childlessed det_ids + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(time_filter); + psFree(detRun); + return false; + } + + if (!detRunInsertObject(config->dbh, detRun)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(time_filter); + psFree(detRun); + return false; + } + psFree(detRun); + + // get the det_id + psS64 newDet_id = psDBLastInsertID(config->dbh); + + psString query = pxDataGet("dettool_definebydetrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (time_filter->list->n) { + psString whereClause = psDBGenerateWhereConditionSQL(time_filter, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(time_filter); + + if (!p_psDBRunQuery(config->dbh, query, (long long) newDet_id, (long long) det_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // print the new det_id + psArray *newDetRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", newDet_id); + newDetRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!newDetRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check the result... we should have only found one det_id + if (psArrayLength(newDetRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 " (this should not happen)", newDet_id); + return false; // unreachable + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, newDetRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(newDetRuns); + return false; + } + psFree(newDetRuns); + + return true; +} + +static bool runsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_type", "det_type", "=="); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-mode", "mode", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + + // airmass_min < airmass < airmass_max + PXOPT_COPY_F32(config->args, where, "-airmass", "airmass_min", "<="); + PXOPT_COPY_F32(config->args, where, "-airmass", "airmass_max", ">="); + + // exp_time_min < exp_time < exp_time_max + PXOPT_COPY_F32(config->args, where, "-exp_time", "exp_time_min", "<="); + PXOPT_COPY_F32(config->args, where, "-exp_time", "exp_time_max", ">="); + + // ccd_temp_min < ccd_temp < ccd_temp_max + PXOPT_COPY_F32(config->args, where, "-ccd_temp", "ccd_temp_min", "<="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp", "ccd_temp_max", ">="); + + PXOPT_COPY_F64(config->args, where, "-posang", "posang_min", "<="); + PXOPT_COPY_F64(config->args, where, "-posang", "posang_max", ">="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(active, config->args, "-active", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psString query = pxDataGet("dettool_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list) && active) { + psString whereClause = psDBGenerateWhereSQL(where, "detRun"); + psStringAppend(&query, " %s AND (detRun.state = 'run' OR detRun.state = 'stop' OR detRun.state = 'register')", whereClause); + psFree(whereClause); + } + if (psListLength(where->list) && !active) { + psString whereClause = psDBGenerateWhereSQL(where, "detRun"); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + if (!psListLength(where->list) && active) { + psStringAppend(&query, " WHERE (detRun.state = 'run' OR detRun.state = 'stop' OR detRun.state = 'register')"); + } + psFree (where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *runs = p_psDBFetchResult(config->dbh); + if (!runs) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(runs)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(runs); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, runs, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(runs); + return false; + } + + psFree(runs); + + return true; +} + +static bool childlessrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_type", "det_type", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("dettool_childlessrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detRun"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree (where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static detInputExpRow *rawDetrenTodetInputExpRow(rawExpRow *rawExp, psS64 det_id, psS32 iteration) +{ + PS_ASSERT_PTR_NON_NULL(rawExp, NULL); + + return detInputExpRowAlloc( + det_id, + iteration, + rawExp->exp_id, + true // use + ); +} + +static bool inputMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + psString query = pxDataGet("dettool_input.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, "detInputExp"); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree (where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detInputExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool rawMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + + psString query = pxDataGet("dettool_raw.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree (where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool updatedetrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_BOOL(again, config->args, "-again", false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + + // either -rerun or -state must be specified + if (!(again || state)) { + psError(PS_ERR_UNKNOWN, true, "either -again or -state must be specified"); + return false; + } + if (again && state) { + psError(PS_ERR_UNKNOWN, true, "-accept and -reject are exclusive"); + return false; + } + + if (state) { + // set detRun.state to state + return setDetRunState(config, det_id, state); + } + + // else + // -again + if (!startNewIteration(config, det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to start new iteration"); + return false; + } + + return true; +} + +// used by updatedetrunMode +bool startNewIteration(pxConfig *config, psS64 det_id) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("dettool_start_new_iteration.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // restrict the update only to the specified det_id + psStringAppend(&query, " WHERE det_id = %" PRId64 , det_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, false, "det_id %" PRId64 " not found", det_id); + psFree(output); + return false; + } + + // start a transaction so we don't end up with an incremented iteration + // count but no detInputExps + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // up the detRuns iteration count for for the single det_id we are + // operating on + // XXX this will have to changed in order to support multiple det_ids with + // a single invocation of this functions + psS32 newIteration = incrementIteration(config, det_id); + if (!newIteration) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + bool status = false; + psS64 exp_id = psMetadataLookupS64(&status, row, "exp_id"); + if (!status) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_id"); + psFree(output); + return false; + } + bool accept = psMetadataLookupBool(&status, row, "accept"); + if (!status) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for accept"); + psFree(output); + return false; + } + + // detResidExp.include is used to set detInputExp.include + if (!detInputExpInsert( + config->dbh, + det_id, + newIteration, + exp_id, + accept + ) + ) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + } + + psFree(output); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool rerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id is required + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + + // we have to support multipe exp_ids + psMetadataItem *item = psMetadataLookup(config->args, "-exp_id"); + if (!item) { + // this shouldn't actually happen when using psArgs + psError(PS_ERR_UNKNOWN, true, "-exp_id is required"); + return false; + } + + psList *exp_id_list = item->data.list; + psMetadata *where = psMetadataAlloc(); + // make sure that -exp_id was parsed correctly + // XXX this can be removed someday + if (item->type == PS_DATA_METADATA_MULTI) { + psListIterator *iter = psListIteratorAlloc(item->data.list, 0, false); + psMetadataItem *mItem = NULL; + while ((mItem = psListGetAndIncrement(iter))) { + psString exp_id = mItem->data.V; + // if exp_id is NULL then it means that -exp_id has not been + // specified + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, + "at least one -exp_id is required"); + psFree(where); + return false; + } + + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", + PS_META_DUPLICATE_OK, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(iter); + psFree(where); + return false; + } + } + psFree(iter); + } else { + psAbort("-exp_id was not parsed correctly (this should not happen"); + } + + // check that the specified exp_ids actually exist in the iteration zero + // detInputExp set + + // add the det_id & iteration == 0 to the where clause + if (!psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return false; + } + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", 0)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + return false; + } + + psArray *detrendExps = detInputExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + if (!detrendExps) { + psError(PS_ERR_UNKNOWN, false, "no rawExp rows found"); + psFree(where); + return false; + } + + // build a hash for the valid exp_ids + psHash *valid_exp_ids = psHashAlloc(psArrayLength(detrendExps)); + for (long i = 0; i < psArrayLength(detrendExps); i++) { + psString exp_idStr = psDBIntToString(((detInputExpRow *)detrendExps->data[i])->exp_id); + psHashAdd(valid_exp_ids, exp_idStr, detrendExps->data[i]); + psFree(exp_idStr); + } + psFree(detrendExps); + + // start a transaction so we don't end up with an incremented iteration + // count but no detInputExps + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(valid_exp_ids); + return false; + } + + // up the detRuns iteration count + psS32 newIteration = incrementIteration(config, det_id); + if (!newIteration) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(valid_exp_ids); + return false; + } + + // check exp_ids and build up an array of new detInputExp rows at the same + // time + psListIterator *iter = psListIteratorAlloc(exp_id_list, 0, false); + psMetadataItem *mItem = NULL; + psArray *newInputExps = psArrayAllocEmpty(psListLength(exp_id_list)); + while ((mItem = psListGetAndIncrement(iter))) { + detInputExpRow *inputExp = psHashLookup(valid_exp_ids, + (char *)mItem->data.V); + if (!inputExp) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + // invalid exp_id + psError(PS_ERR_UNKNOWN, false, "exp_id %s is invalid for det_id %" PRId64, + (char *)mItem->data.V, det_id); + psFree(iter); + psFree(valid_exp_ids); + return false; + } + detInputExpRow *newInputExp = detInputExpRowAlloc( + det_id, + newIteration, + inputExp->exp_id, + true // use + ); + psArrayAdd(newInputExps, 0, newInputExp); + psFree(newInputExp); + } + psFree(iter); + psFree(valid_exp_ids); + + for (long i = 0; i < psArrayLength(newInputExps); i++) { + if (!detInputExpInsertObject(config->dbh, newInputExps->data[i])) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(newInputExps); + return false; + } + } + psFree(newInputExps); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool register_detrendMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(det_type, config->args, "-det_type", true, false); // required + PXOPT_LOOKUP_STR(filelevel, config->args, "-filelevel", true, false); // required + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); + PXOPT_LOOKUP_STR(camera, config->args, "-inst", false, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", false, false); + PXOPT_LOOKUP_STR(exp_type, config->args, "-exp_type", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-filter", false, false); + PXOPT_LOOKUP_F32(airmass_min, config->args, "-airmass_min", false, false); + PXOPT_LOOKUP_F32(airmass_max, config->args, "-airmass_max", false, false); + PXOPT_LOOKUP_F32(exp_time_min, config->args, "-exp_time_min", false, false); + PXOPT_LOOKUP_F32(exp_time_max, config->args, "-exp_time_max", false, false); + PXOPT_LOOKUP_F32(ccd_temp_min, config->args, "-ccd_temp_min", false, false); + PXOPT_LOOKUP_F32(ccd_temp_max, config->args, "-ccd_temp_max", false, false); + PXOPT_LOOKUP_F64(posang_min, config->args, "-posang_min", false, false); + PXOPT_LOOKUP_F64(posang_max, config->args, "-posang_max", false, false); + PXOPT_LOOKUP_F64(solang_min, config->args, "-solang_min", false, false); + PXOPT_LOOKUP_F64(solang_max, config->args, "-solang_max", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_TIME(time_begin, config->args, "-time_begin", false, false); + PXOPT_LOOKUP_TIME(time_end, config->args, "-time_end", false, false); + PXOPT_LOOKUP_TIME(use_begin, config->args, "-use_begin", false, false); + PXOPT_LOOKUP_TIME(use_end, config->args, "-use_end", false, false); + PXOPT_LOOKUP_S64(ref_det_id, config->args, "-ref_det_id", false, false); + PXOPT_LOOKUP_S32(ref_iter, config->args, "-ref_iter", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!detRunInsert(config->dbh, + 0, // det_id + 0, // the iteration is fixed at 0 + det_type, + "register", // mode is required to be "register" + "register", // state + filelevel, + workdir, + camera, + telescope, + exp_type, + NULL, + filter, + airmass_min, + airmass_max, + exp_time_min, + exp_time_max, + ccd_temp_min, + ccd_temp_max, + posang_min, + posang_max, + registered, + time_begin, + time_end, + use_begin, + use_end, + solang_min, + solang_max, + label, + ref_det_id, + ref_iter + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + // print the new detRun + psS64 det_id = psDBLastInsertID(config->dbh); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id); + detRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check results + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 "(this should not happen)", det_id); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, detRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detRuns); + return false; + } + psFree(detRuns); + + return true; +} + +// NOTE : this function is also used by addcorrectimfileMode +bool register_detrend_imfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); // required + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); // required + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + if (!detRegisteredImfileInsert(config->dbh, + det_id, + 0, // the iteration is fixed at 0 + class_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + "full", + 0 // fault code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static psS32 incrementIteration(pxConfig *config, psS64 det_id) +{ + // this function returns zero on error + PS_ASSERT_PTR_NON_NULL(config, 0); + + char *query = "UPDATE detRun SET iteration = iteration + 1 WHERE det_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to increment iteration for det_id %" PRId64, det_id); + return 0; + } + + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return 0; + } + + psArray *detRuns = detRunSelectRowObjects(config->dbh, where, 0); + psFree(where); + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "no detRun rows found"); + return 0; + } + + // sanity check the database + if (psArrayLength(detRuns) != 1) { + // this should no happen + psAbort( "database query return too many rows (this should not happen"); + } + + psS32 newIteration = ((detRunRow *)detRuns->data[0])->iteration; + psFree(detRuns); + + return newIteration; +} + +bool setDetRunState(pxConfig *config, psS64 det_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "drop", 5) == 0) + || (strncmp(state, "register", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid detRun state: %s", state); + return false; + } + + char *query = "UPDATE detRun SET state = '%s' WHERE det_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64, det_id); + return false; + } + + return true; +} + +bool isValidDataState (const char *data_state) { + + // check that state is a valid string value + if (!strncmp(data_state, "run", 4)) return true; + if (!strncmp(data_state, "stop", 5)) return true; + if (!strncmp(data_state, "drop", 5)) return true; + if (!strncmp(data_state, "register", 4)) return true; + + psError(PS_ERR_UNKNOWN, true, "invalid data state: %s", data_state); + return false; +} + +bool isValidMode(pxConfig *config, const char *mode) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(mode, false); + + // check that state is a valid string value + if (!strncmp(mode, "master", 7)) return true; + if (!strncmp(mode, "verify", 7)) return true; + + psError(PS_ERR_UNKNOWN, false, "invalid detRun mode: %s", mode); + return false; +} + + +bool setProcessedImfileDataState(pxConfig *config, psS64 det_id, psS64 exp_id, const char *class_id, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + PS_ASSERT_PTR_NON_NULL(class_id, false); + + if (!isValidDataState (data_state)) return false; + + char *query = "UPDATE detProcessedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND exp_id = %" PRId64 + " AND class_id = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, data_state, det_id, exp_id, class_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", exp_id %" PRId64 ", class_id %s", + det_id, exp_id, class_id); + return false; + } + + return true; +} + +bool setProcessedExpDataState(pxConfig *config, psS64 det_id, psS64 exp_id, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + + if (!isValidDataState (data_state)) return false; + + char *query = "UPDATE detProcessedExp SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND exp_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, data_state, det_id, exp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", exp_id %" PRId64, + det_id, exp_id); + return false; + } + + return true; +} + + +bool setStackedImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(class_id, false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + + if (!isValidDataState (data_state)) return false; + + char *query = "UPDATE detStackedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32 + " AND class_id = %s"; + if (!p_psDBRunQuery(config->dbh, query, data_state, det_id, iteration, class_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", iteration %" PRId32 "class_id %s", + det_id, iteration, class_id); + return false; + } + + return true; +} + +bool setNormStatImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(class_id, false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + + if (!isValidDataState (data_state)) return false; + + char *query = "UPDATE detNormalizedStatImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32 + " AND class_id = %s"; + if (!p_psDBRunQuery(config->dbh, query, data_state, det_id, iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", iteration %" PRId32, + det_id, iteration); + return false; + } + + return true; +} + +bool setNormImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(class_id, false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + + if (!isValidDataState (data_state)) return false; + + char *query = "UPDATE detNormalizedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32 + " AND class_id = %s"; + if (!p_psDBRunQuery(config->dbh, query, data_state, det_id, iteration, class_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", iteration %" PRId32 " class %s", + det_id, iteration, class_id); + return false; + } + + return true; +} + +bool setNormExpDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + + if (!isValidDataState (data_state)) return false; + + char *query = "UPDATE detNormalizedExp SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQuery(config->dbh, query, data_state, det_id, iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", iteration %" PRId32, + det_id, iteration); + return false; + } + + return true; +} + +bool setResidImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, psS64 exp_id, const char *class_id, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + PS_ASSERT_PTR_NON_NULL(class_id, false); + + if (!isValidDataState (data_state)) return false; + + char *query = "UPDATE detResidImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32 + " AND exp_id = %" PRId64 + " AND class_id = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, data_state, det_id, iteration, exp_id, class_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", iteration %" PRId32 ", exp_id %" PRId64 ", class_id %s", + det_id, iteration, exp_id, class_id); + return false; + } + + return true; +} + +bool setResidExpDataState(pxConfig *config, psS64 det_id, psS32 iteration, psS64 exp_id, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + + if (!isValidDataState (data_state)) return false; + + char *query = "UPDATE detResidExp SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32 + " AND exp_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, data_state, det_id, iteration, exp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", iteration %" PRId32 ", exp_id %" PRId64, + det_id, iteration, exp_id); + return false; + } + + return true; +} + +#if 0 +// XXX this function was left in commented as this method may be useful in the +// future +static psArray *validDetInputClassIds(pxConfig *config, const char *det_id) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + // det_id is input as a string because the fact that it is an integer + // is just a database impliementation detail. + PS_ASSERT_PTR_NON_NULL(det_id, NULL); + + psArray *rawImfiles = searchInputImfiles(config, det_id); + if (!rawImfiles) { + return NULL; + } + + psArray *valid_class_ids = NULL; + { + // All this jumping through hoops is so that we end up with unique + // values. PP thinks that making multiple passes through this array and + // deleting matched elements would end up being more expensive then a + // double sort and stagger scheme. JH thinks it would be cheaper to + // just do a unqiue sort and delete. So this is really just a cheap + // hack to avoid implimenting a unique sort function but at least it's + // stable. + psHash *hash = psHashAlloc(psArrayLength(rawImfiles)); + for (long i = 0; i < psArrayLength(rawImfiles); i++) { + psHashAdd(hash, + ((rawImfileRow *)rawImfiles->data[i])->class_id, + ((rawImfileRow *)rawImfiles->data[i])->class_id + ); + } + valid_class_ids = psHashToArray(hash); + psFree(hash); + } + psFree(rawImfiles); + + return valid_class_ids; +} + +static psArray *searchInputImfiles(pxConfig *config, const char *det_id) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + // det_id is input as a string because the fact that it is an integer + // is just a database impliementation detail. + PS_ASSERT_PTR_NON_NULL(det_id, NULL); + + psArray *inputExps = NULL; + { + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddS32(where, PS_LIST_TAIL, "det_id", 0, "==", + (psS32)atoi(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return NULL; + } + inputExps = detInputExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + } + if (!inputExps) { + psError(PS_ERR_UNKNOWN, false, "no detInputExp rows found"); + return NULL; + } + + // find rawImfiles associated with detInputExps + psArray *rawImfiles = NULL; + { + psMetadata *where = psMetadataAlloc(); + for (long i = 0; i < psArrayLength(inputExps); i++) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", + PS_META_DUPLICATE_OK, "==", + ((detInputExpRow *)inputExps->data[i])->exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(inputExps); + psFree(where); + return NULL; + } + } + psFree(inputExps); + rawImfiles = rawImfileSelectRowObjects(config->dbh, where, 0); + // XXX this really should be sorted for uniqueness + psFree(where); + } + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found"); + return NULL; + } + + return rawImfiles; +} +#endif + Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool.h (revision 22161) @@ -0,0 +1,225 @@ +/* + * dettool.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#ifndef DETTOOL_H +#define DETTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + DETTOOL_MODE_NONE = PXTOOL_MODE_NONE, + DETTOOL_MODE_PENDING, + DETTOOL_MODE_DEFINEBYTAG, + DETTOOL_MODE_DEFINEBYQUERY, + DETTOOL_MODE_DEFINEBYDETRUN, + DETTOOL_MODE_MAKECORRECTION, + DETTOOL_MODE_TOCORRECTEXP, + DETTOOL_MODE_TOCORRECTIMFILE, + DETTOOL_MODE_ADDCORRECTIMFILE, + DETTOOL_MODE_RUNS, + DETTOOL_MODE_CHILDLESSRUN, + DETTOOL_MODE_INPUT, + DETTOOL_MODE_RAW, + + DETTOOL_MODE_TOPROCESSEDIMFILE, + DETTOOL_MODE_ADDPROCESSEDIMFILE, + DETTOOL_MODE_PROCESSEDIMFILE, + DETTOOL_MODE_REVERTPROCESSEDIMFILE, + DETTOOL_MODE_UPDATEPROCESSEDIMFILE, + DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDIMFILE, + DETTOOL_MODE_DONECLEANUP_PROCESSEDIMFILE, + + DETTOOL_MODE_TOPROCESSEDEXP, + DETTOOL_MODE_ADDPROCESSEDEXP, + DETTOOL_MODE_PROCESSEDEXP, + DETTOOL_MODE_REVERTPROCESSEDEXP, + DETTOOL_MODE_UPDATEPROCESSEDEXP, + DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDEXP, + DETTOOL_MODE_DONECLEANUP_PROCESSEDEXP, + + DETTOOL_MODE_TOSTACKED, + DETTOOL_MODE_ADDSTACKED, + DETTOOL_MODE_STACKED, + DETTOOL_MODE_REVERTSTACKED, + DETTOOL_MODE_UPDATESTACKED, + DETTOOL_MODE_PENDINGCLEANUP_STACKED, + DETTOOL_MODE_DONECLEANUP_STACKED, + + DETTOOL_MODE_TONORMALIZEDSTAT, + DETTOOL_MODE_ADDNORMALIZEDSTAT, + DETTOOL_MODE_NORMALIZEDSTAT, + DETTOOL_MODE_REVERTNORMALIZEDSTAT, + DETTOOL_MODE_UPDATENORMALIZEDSTAT, + DETTOOL_MODE_PENDINGCLEANUP_NORMALIZEDSTAT, + DETTOOL_MODE_DONECLEANUP_NORMALIZEDSTAT, + + DETTOOL_MODE_TONORMALIZE, + DETTOOL_MODE_ADDNORMALIZEDIMFILE, + DETTOOL_MODE_NORMALIZEDIMFILE, + DETTOOL_MODE_REVERTNORMALIZEDIMFILE, + DETTOOL_MODE_UPDATENORMALIZEDIMFILE, + DETTOOL_MODE_PENDINGCLEANUP_NORMALIZEDIMFILE, + DETTOOL_MODE_DONECLEANUP_NORMALIZEDIMFILE, + + DETTOOL_MODE_TONORMALIZEDEXP, + DETTOOL_MODE_ADDNORMALIZEDEXP, + DETTOOL_MODE_NORMALIZEDEXP, + DETTOOL_MODE_REVERTNORMALIZEDEXP, + DETTOOL_MODE_UPDATENORMALIZEDEXP, + DETTOOL_MODE_PENDINGCLEANUP_NORMALIZEDEXP, + DETTOOL_MODE_DONECLEANUP_NORMALIZEDEXP, + + DETTOOL_MODE_TORESIDIMFILE, + DETTOOL_MODE_ADDRESIDIMFILE, + DETTOOL_MODE_RESIDIMFILE, + DETTOOL_MODE_REVERTRESIDIMFILE, + DETTOOL_MODE_UPDATERESIDIMFILE, + DETTOOL_MODE_PENDINGCLEANUP_RESIDIMFILE, + DETTOOL_MODE_DONECLEANUP_RESIDIMFILE, + + DETTOOL_MODE_TORESIDEXP, + DETTOOL_MODE_ADDRESIDEXP, + DETTOOL_MODE_RESIDEXP, + DETTOOL_MODE_REVERTRESIDEXP, + DETTOOL_MODE_UPDATERESIDEXP, + DETTOOL_MODE_PENDINGCLEANUP_RESIDEXP, + DETTOOL_MODE_DONECLEANUP_RESIDEXP, + + DETTOOL_MODE_TODETRUNSUMMARY, + DETTOOL_MODE_ADDDETRUNSUMMARY, + DETTOOL_MODE_DETRUNSUMMARY, + DETTOOL_MODE_REVERTDETRUNSUMMARY, + DETTOOL_MODE_UPDATEDETRUNSUMMARY, + DETTOOL_MODE_UPDATEDETRUN, + DETTOOL_MODE_RERUN, + DETTOOL_MODE_REGISTER_DETREND, + DETTOOL_MODE_REGISTER_DETREND_IMFILE +} dettoolMode; + +pxConfig *dettoolConfig(pxConfig *config, int argc, char **argv); + +// correct +bool makecorrectionMode(pxConfig *config); +bool tocorrectexpMode(pxConfig *config); +bool tocorrectimfileMode(pxConfig *config); +bool addcorrectimfileMode(pxConfig *config); + +// register +bool register_detrend_imfileMode(pxConfig *config); + +// processedimfile +bool toprocessedimfileMode(pxConfig *config); +bool addprocessedimfileMode(pxConfig *config); +bool processedimfileMode(pxConfig *config); +bool revertprocessedimfileMode(pxConfig *config); +bool updateprocessedimfileMode(pxConfig *config); +bool pendingcleanup_processedimfileMode(pxConfig *config); +bool donecleanup_processedimfileMode(pxConfig *config); + +// processedexp +bool toprocessedexpMode(pxConfig *config); +bool addprocessedexpMode(pxConfig *config); +bool processedexpMode(pxConfig *config); +bool revertprocessedexpMode(pxConfig *config); +bool updateprocessedexpMode(pxConfig *config); +bool pendingcleanup_processedexpMode(pxConfig *config); +bool donecleanup_processedexpMode(pxConfig *config); + +// stackedimfile +bool tostackedMode(pxConfig *config); +bool addstackedMode(pxConfig *config); +bool stackedMode(pxConfig *config); +bool revertstackedMode(pxConfig *config); +bool updatestackedMode(pxConfig *config); +bool pendingcleanup_stackedMode(pxConfig *config); +bool donecleanup_stackedMode(pxConfig *config); + +// normalizedstat +bool tonormalizedstatMode(pxConfig *config); +bool addnormalizedstatMode(pxConfig *config); +bool normalizedstatMode(pxConfig *config); +bool revertnormalizedstatMode(pxConfig *config); +bool updatenormalizedstatMode(pxConfig *config); +bool pendingcleanup_normalizedstatMode(pxConfig *config); +bool donecleanup_normalizedstatMode(pxConfig *config); + +// normalizedimfile +bool tonormalizeMode(pxConfig *config); +bool addnormalizedimfileMode(pxConfig *config); +bool normalizedimfileMode(pxConfig *config); +bool revertnormalizedimfileMode(pxConfig *config); +bool updatenormalizedimfileMode(pxConfig *config); +bool pendingcleanup_normalizedimfileMode(pxConfig *config); +bool donecleanup_normalizedimfileMode(pxConfig *config); + +// normalizedexp +bool tonormalizedexpMode(pxConfig *config); +bool addnormalizedexpMode(pxConfig *config); +bool normalizedexpMode(pxConfig *config); +bool revertnormalizedexpMode(pxConfig *config); +bool updatenormalizedexpMode(pxConfig *config); +bool pendingcleanup_normalizedexpMode(pxConfig *config); +bool donecleanup_normalizedexpMode(pxConfig *config); + +// residimfile +bool toresidimfileMode(pxConfig *config); +bool addresidimfileMode(pxConfig *config); +bool residimfileMode(pxConfig *config); +bool revertresidimfileMode(pxConfig *config); +bool updateresidimfileMode(pxConfig *config); +bool pendingcleanup_residimfileMode(pxConfig *config); +bool donecleanup_residimfileMode(pxConfig *config); + +// residexp +bool toresidexpMode(pxConfig *config); +bool addresidexpMode(pxConfig *config); +bool residexpMode(pxConfig *config); +bool revertresidexpMode(pxConfig *config); +bool updateresidexpMode(pxConfig *config); +bool pendingcleanup_residexpMode(pxConfig *config); +bool donecleanup_residexpMode(pxConfig *config); + +// detrunsummary +bool todetrunsummaryMode(pxConfig *config); +bool adddetrunsummaryMode(pxConfig *config); +bool detrunsummaryMode(pxConfig *config); +bool revertdetrunsummaryMode(pxConfig *config); +bool updatedetrunsummaryMode(pxConfig *config); + +// other utilities +bool startNewIteration(pxConfig *config, psS64 det_id); +bool setDetRunState(pxConfig *config, psS64 det_id, const char *state); +bool isValidDataState (const char *data_state); +bool isValidMode(pxConfig *config, const char *mode); + +// functions to set the 'data_state' for stages +bool setProcessedImfileDataState(pxConfig *config, psS64 det_id, psS64 exp_id, const char *class_id, const char *data_state); +bool setProcessedExpDataState(pxConfig *config, psS64 det_id, psS64 exp_id, const char *data_state); +bool setStackedImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state); +bool setNormStatImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state); +bool setNormImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state); +bool setNormExpDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *data_state); +bool setResidImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, psS64 exp_id, const char *class_id, const char *data_state); +bool setResidExpDataState(pxConfig *config, psS64 det_id, psS32 iteration, psS64 exp_id, const char *data_state); + +#endif // DETTOOL_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/dettoolConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/dettoolConfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/dettoolConfig.c (revision 22161) @@ -0,0 +1,962 @@ +/* + * dettoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +pxConfig *dettoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -pending + // XXX EAM : is this used? does it make sense? + psMetadata *pendingArgs = psMetadataAlloc(); + psMetadataAddS64(pendingArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exposure type", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst", 0, "search by camera", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope", 0, "search by telescope", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-uri", 0, "search by URL", NULL); + psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -definebytag + psMetadata *definebytagArgs = psMetadataAlloc(); + psMetadataAddS64(definebytagArgs, PS_LIST_TAIL, "-exp_id", PS_META_DUPLICATE_OK, "include this exposure (multiple OK, required)", 0); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-det_type", 0, "define the type of detrend run (required)", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-mode", 0, "define the mode of this detrend run", "master"); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-filelevel", 0, "define filelevel", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-inst", 0, "define camera", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-exp_type", 0, "define exposure type", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-filter", 0, "define filter ", NULL); + psMetadataAddF32(definebytagArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(definebytagArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF32(definebytagArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min exposure time", NAN); + psMetadataAddF32(definebytagArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max exposure time", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddTime(definebytagArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddTime(definebytagArgs, PS_LIST_TAIL, "-time_begin", 0, "detrend applies to exposures taken during this period", NULL); + psMetadataAddTime(definebytagArgs, PS_LIST_TAIL, "-time_end", 0, "detrend applies to exposures taken during this period", NULL); + psMetadataAddTime(definebytagArgs, PS_LIST_TAIL, "-use_begin", 0, "start of detrend run applicable period", NULL); + psMetadataAddTime(definebytagArgs, PS_LIST_TAIL, "-use_end", 0, "end of detrend run applicable period", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class for processing", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-label", 0, "define detrun label", NULL); + psMetadataAddBool(definebytagArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psMetadataAddS64(definebytagArgs, PS_LIST_TAIL, "-ref_det_id", 0, "define reference detrend (verify mode only)", 0); + psMetadataAddS32(definebytagArgs, PS_LIST_TAIL, "-ref_iter", 0, "define reference detrend (verify mode only)", -1); + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-det_type", 0, "define the type of detrend run (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-mode", 0, "define the mode of this detrend run", "master"); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filelevel", 0, "define filelevel", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-inst", 0, "define camera (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filter", 0, "define filter ", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min exposure time", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max exposure time", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-time_begin", 0, "detrend applies to exposures taken during this period", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-time_end", 0, "detrend applies to exposures taken during this period", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-use_begin", 0, "start of detrend run applicable period", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-use_end", 0, "end of detrend run applicable period", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_type", 0, "search for exp_type", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_inst", 0, "search for camera", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_telescope", 0, "search for telescope", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", 0, "search for filter", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_uri", 0, "search for uri", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-select_dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-select_dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_airmass_max", 0, "define max airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_min", 0, "define min exposure time", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_max", 0, "define max exposure time", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_pon_time_min", 0, "define min power-on time", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_pon_time_max", 0, "define max power-on time", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_min", 0, "define min solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_max", 0, "define max solar angle", NAN); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "print the exposures that would be included in the detrend run and exit", false); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class for processing", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define detrun label", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-random_subset", 0, "use a random subset of elements", false); + psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random_limit", 0, "use this number of random elements", 20); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-ref_det_id", 0, "define reference detrend (verify mode only)", 0); + psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-ref_iter", 0, "define reference detrend (verify mode only)", -1); + + // -definebydetrun + psMetadata *definebydetrunArgs = psMetadataAlloc(); + psMetadataAddS64(definebydetrunArgs, PS_LIST_TAIL, "-det_id", 0, "det ID to base a new detRun on (required)", 0); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_det_type", 0, "define the type of detrend run", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_mode", 0, "define the mode of this detrend run", "master"); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_exp_type", 0, "define exposure type", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_filelevel", 0, "define filelevel", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_filter", 0, "define filter ", NULL); + psMetadataAddF32(definebydetrunArgs, PS_LIST_TAIL, "-set_airmass_min", 0, "define airmass", NAN); + psMetadataAddF32(definebydetrunArgs, PS_LIST_TAIL, "-set_airmass_max", 0, "define airmass", NAN); + psMetadataAddF32(definebydetrunArgs, PS_LIST_TAIL, "-set_exp_time_min", 0, "define exposure time", NAN); + psMetadataAddF32(definebydetrunArgs, PS_LIST_TAIL, "-set_exp_time_max", 0, "define exposure time", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_ccd_temp_min", 0, "define ccd tempature", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_ccd_temp_max", 0, "define ccd tempature", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_posang_min", 0, "define rotator position angle", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_posang_max", 0, "define rotator position angle", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_solang_min", 0, "define solar angle", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_solang_max", 0, "define solar angle", NAN); + psMetadataAddTime(definebydetrunArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddTime(definebydetrunArgs, PS_LIST_TAIL, "-set_time_begin", 0, "start of period to apply detrend too", NULL); + psMetadataAddTime(definebydetrunArgs, PS_LIST_TAIL, "-set_time_end", 0, "end of period to apply detrend too", NULL); + psMetadataAddTime(definebydetrunArgs, PS_LIST_TAIL, "-set_use_begin", 0, "start of detrend run applicable period", NULL); + psMetadataAddTime(definebydetrunArgs, PS_LIST_TAIL, "-set_use_end", 0, "end of detrend run applicable period", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class for processing", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_label", 0, "define detrun label", NULL); + psMetadataAddBool(definebydetrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_input_begin", 0, "require input exp to be in this period", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_input_end", 0, "require input exp to be in this period", NULL); + + // *** these are in dettool_correction.c *** + // -makecorrection + psMetadata *makecorrectionArgs = psMetadataAlloc(); + psMetadataAddS64(makecorrectionArgs, PS_LIST_TAIL, "-det_id", 0, "detRun det_id to be corrected (required)", 0); + psMetadataAddS32(makecorrectionArgs, PS_LIST_TAIL, "-iteration", 0, "detRun iteration to be corrected (required)", -1);; + psMetadataAddBool(makecorrectionArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -tocorrectexp + psMetadata *tocorrectexpArgs = psMetadataAlloc(); + psMetadataAddU64(tocorrectexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tocorrectexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -tocorrectimfile + psMetadata *tocorrectimfileArgs = psMetadataAlloc(); + psMetadataAddS64(tocorrectimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddU64(tocorrectimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tocorrectimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addcorrectimfile + psMetadata *addcorrectimfileArgs = psMetadataAlloc(); + psMetadataAddS64(addcorrectimfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID (required)", NULL); + psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-uri", 0, "define resid file URI", NULL); + psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + + // -runs + psMetadata *runsArgs = psMetadataAlloc(); + psMetadataAddStr(runsArgs, PS_LIST_TAIL, "-det_type", 0, "search for type of detrend run", NULL); + psMetadataAddS64(runsArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddStr(runsArgs, PS_LIST_TAIL, "-inst", 0, "search for camera (instrument)", NULL); + psMetadataAddStr(runsArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddStr(runsArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(runsArgs, PS_LIST_TAIL, "-airmass", 0, "match airmass", NAN); + psMetadataAddF32(runsArgs, PS_LIST_TAIL, "-exp_time", 0, "match exp time", NAN); + psMetadataAddF32(runsArgs, PS_LIST_TAIL, "-ccd_temp", 0, "match ccd temp", NAN); + psMetadataAddF64(runsArgs, PS_LIST_TAIL, "-posang", 0, "match posang", NAN); + psMetadataAddStr(runsArgs, PS_LIST_TAIL, "-mode", 0, "search for det run by mode", NULL); + psMetadataAddBool(runsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(runsArgs, PS_LIST_TAIL, "-active", 0, "only return active detRuns", false); + psMetadataAddU64(runsArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -childlessrun + psMetadata *childlessrunArgs = psMetadataAlloc(); + psMetadataAddStr(childlessrunArgs, PS_LIST_TAIL, "-det_type", 0, "search for type of detrend run", NULL); + psMetadataAddU64(childlessrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(childlessrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -input + psMetadata *inputArgs = psMetadataAlloc(); + psMetadataAddS64(inputArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddS32(inputArgs, PS_LIST_TAIL, "-iteration", 0, "search for iteration number (default 0)", 0); + psMetadataAddS64(inputArgs, PS_LIST_TAIL, "-exp_id", 0, "search for exp ID", 0); + psMetadataAddBool(inputArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -raw + psMetadata *rawArgs = psMetadataAlloc(); + psMetadataAddBool(rawArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -toprocessedimfile + psMetadata *toprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(toprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddS64(toprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search for exp ID", 0); + psMetadataAddStr(toprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID", NULL); + psMetadataAddU64(toprocessedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(toprocessedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addprocessedimfile + psMetadata *addprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define exp ID (required)", 0); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID (required)", NULL); + psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-recip", 0, "define recipe", NULL); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_0", 0, "define fringe slope (0th term)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_1", 0, "define fringe slope (1st term)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_2", 0, "define fringe slope (2nd term)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + + // -processedimfile + psMetadata *processedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search for exp ID", 0); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-select_state", 0, "search for state", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-select_mode", 0, "search for mode", NULL); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-included", 0, "restrict results to exposures 'includeded' in the current iteration", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertprocessedimfile + psMetadata *revertprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID (required)", 0); + psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddS16(revertprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updateprocessedimfile + psMetadata *updateprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + + // -pendingcleanup_processedimfile + psMetadata *pendingcleanup_processedimfileArgs = psMetadataAlloc(); + psMetadataAddBool(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + + // -donecleanup_processedimfile + psMetadata *donecleanup_processedimfileArgs = psMetadataAlloc(); + psMetadataAddBool(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + + // -toprocessedexp + psMetadata *toprocessedexpArgs = psMetadataAlloc(); + psMetadataAddU64(toprocessedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(toprocessedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addproccessedexp + psMetadata *addprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "define detrend ID (required)", 0); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-recip", 0, "define recipe", NULL); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_0", 0, "define fringe slope (0th term)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_1", 0, "define fringe slope (1st term)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_2", 0, "define fringe slope (2nd term)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -proccessedexp + psMetadata *processedexpArgs = psMetadataAlloc(); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID", 0); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp ID", 0); + psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertprocessedexp + psMetadata *revertprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updateprocessedexp + psMetadata *updateprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + + // -pendingcleanup_processedexp + psMetadata *pendingcleanup_processedexpArgs = psMetadataAlloc(); + psMetadataAddBool(pendingcleanup_processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanup_processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(pendingcleanup_processedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(pendingcleanup_processedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + + // -donecleanup_processedexp + psMetadata *donecleanup_processedexpArgs = psMetadataAlloc(); + psMetadataAddBool(donecleanup_processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanup_processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(donecleanup_processedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(donecleanup_processedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + + // -tostacked + psMetadata *tostackedArgs = psMetadataAlloc(); + psMetadataAddU64(tostackedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tostackedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addstacked + psMetadata *addstackedArgs = psMetadataAlloc(); + psMetadataAddS64(addstackedArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddS32(addstackedArgs, PS_LIST_TAIL, "-iteration", 0, "define iteration number (required)", 0); + psMetadataAddStr(addstackedArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID (required)", NULL); + psMetadataAddS16(addstackedArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddStr(addstackedArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL); + psMetadataAddStr(addstackedArgs, PS_LIST_TAIL, "-recip", 0, "define recipe", NULL); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + + // -stacked + psMetadata *stackedArgs = psMetadataAlloc(); + psMetadataAddS64(stackedArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddS32(stackedArgs, PS_LIST_TAIL, "-iteration", 0, "search for iteration number", 0); + psMetadataAddStr(stackedArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID", NULL); + psMetadataAddStr(stackedArgs, PS_LIST_TAIL, "-recip", 0, "search for recipe", NULL); + psMetadataAddU64(stackedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(stackedArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(stackedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertstacked + psMetadata *revertstackedArgs= psMetadataAlloc(); + psMetadataAddS64(revertstackedArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID (required)", 0); + psMetadataAddS32(revertstackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(revertstackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddS16(revertstackedArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updatestacked + psMetadata *updatestackedArgs = psMetadataAlloc(); + psMetadataAddS64(updatestackedArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(updatestackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + + // -pendingcleanup_stacked + psMetadata *pendingcleanup_stackedArgs = psMetadataAlloc(); + psMetadataAddBool(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + + // -donecleanup_stacked + psMetadata *donecleanup_stackedArgs = psMetadataAlloc(); + psMetadataAddBool(donecleanup_stackedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanup_stackedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(donecleanup_stackedArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(donecleanup_stackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(donecleanup_stackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + + // -tonormalizedstat + psMetadata *tonormalizedstatArgs = psMetadataAlloc(); + psMetadataAddU64(tonormalizedstatArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tonormalizedstatArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addnormalizedstat + psMetadata *addnormstatArgs = psMetadataAlloc(); + psMetadataAddS64(addnormstatArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddS32(addnormstatArgs, PS_LIST_TAIL, "-iteration", 0, "define iteration number", 0); + psMetadataAddStr(addnormstatArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID (required)", NULL); + psMetadataAddF32(addnormstatArgs, PS_LIST_TAIL, "-norm", 0, "define normal value (required)", NAN); + psMetadataAddS16(addnormstatArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -normalizedstat + psMetadata *normalizedstatArgs = psMetadataAlloc(); + psMetadataAddS64(normalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID", 0); + psMetadataAddS32(normalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(normalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddU64(normalizedstatArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(normalizedstatArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(normalizedstatArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertnormalizedstat + psMetadata *revertnormalizedstatArgs= psMetadataAlloc(); + psMetadataAddS64(revertnormalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(revertnormalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(revertnormalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddS16(revertnormalizedstatArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updatenormalizedstat + psMetadata *updatenormalizedstatArgs = psMetadataAlloc(); + psMetadataAddS64(updatenormalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(updatenormalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + + // -pendingcleanup_normalizedstat + psMetadata *pendingcleanup_normalizedstatArgs = psMetadataAlloc(); + psMetadataAddBool(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + + // -donecleanup_normalizedstat + psMetadata *donecleanup_normalizedstatArgs = psMetadataAlloc(); + psMetadataAddBool(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + + // -tonormalize + psMetadata *tonormalizeArgs = psMetadataAlloc(); + psMetadataAddU64(tonormalizeArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tonormalizeArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addnormalizedimfile + psMetadata *addnormalizedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(addnormalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddS32(addnormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "define iteration number", 0); + psMetadataAddStr(addnormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID (required)", NULL); + psMetadataAddS16(addnormalizedimfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddStr(addnormalizedimfileArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + psMetadataAddStr(addnormalizedimfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + + // -normalizedimfile + psMetadata *normalizedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(normalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddS32(normalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search for iteration number", 0); + psMetadataAddStr(normalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID", NULL); + psMetadataAddStr(normalizedimfileArgs, PS_LIST_TAIL, "-recip", 0, "search for recipe", NULL); + psMetadataAddU64(normalizedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(normalizedimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(normalizedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertnormalizedimfile + psMetadata *revertnormalizedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(revertnormalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(revertnormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(revertnormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddS16(revertnormalizedimfileArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updatenormalizedimfile + psMetadata *updatenormalizedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(updatenormalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(updatenormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + + // -pendingcleanup_normalizedimfile + psMetadata *pendingcleanup_normalizedimfileArgs = psMetadataAlloc(); + psMetadataAddBool(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + + // -donecleanup_normalizedimfile + psMetadata *donecleanup_normalizedimfileArgs = psMetadataAlloc(); + psMetadataAddBool(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + + // -tonormalizedexp + psMetadata *tonormalizedexpArgs = psMetadataAlloc(); + psMetadataAddU64(tonormalizedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tonormalizedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addnormalizedexp + psMetadata *addnormalizedexpArgs = psMetadataAlloc(); + psMetadataAddS64(addnormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddS32(addnormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "define iteration number", 0); + psMetadataAddS16(addnormalizedexpArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddStr(addnormalizedexpArgs, PS_LIST_TAIL, "-recip", 0, "search for recipe", NULL); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + psMetadataAddStr(addnormalizedexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + + // -normalizedexp + psMetadata *normalizedexpArgs = psMetadataAlloc(); + psMetadataAddS64(normalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID", 0); + psMetadataAddS32(normalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(normalizedexpArgs, PS_LIST_TAIL, "-recip", 0, "search for recipe", NULL); + psMetadataAddU64(normalizedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(normalizedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(normalizedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertnormalizedexp + psMetadata *revertnormalizedexpArgs = psMetadataAlloc(); + psMetadataAddS64(revertnormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(revertnormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddS16(revertnormalizedexpArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updatenormalizedexp + psMetadata *updatenormalizedexpArgs = psMetadataAlloc(); + psMetadataAddS64(updatenormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(updatenormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + + // -pendingcleanup_normalizedexp + psMetadata *pendingcleanup_normalizedexpArgs = psMetadataAlloc(); + psMetadataAddBool(pendingcleanup_normalizedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanup_normalizedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(pendingcleanup_normalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(pendingcleanup_normalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + + // -donecleanup_normalizedexp + psMetadata *donecleanup_normalizedexpArgs = psMetadataAlloc(); + psMetadataAddBool(donecleanup_normalizedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanup_normalizedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(donecleanup_normalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(donecleanup_normalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + + // -toresidimfile + psMetadata *toresidimfileArgs = psMetadataAlloc(); + psMetadataAddU64(toresidimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(toresidimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addresidimfile + // XXX does a default value of 0 mean the entry is not instatiated? what about a supplied value of 0? + psMetadata *addresidimfileArgs = psMetadataAlloc(); + psMetadataAddS64(addresidimfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddS32(addresidimfileArgs, PS_LIST_TAIL, "-iteration", 0, "define iteration number", 0); + psMetadataAddS64(addresidimfileArgs, PS_LIST_TAIL, "-ref_det_id", 0, "define reference detrend ID (required)", 0); + psMetadataAddS32(addresidimfileArgs, PS_LIST_TAIL, "-ref_iter", 0, "define reference iteration number (required)", -1); + psMetadataAddS64(addresidimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define detrend ID (required)", 0); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID (required)", NULL); + psMetadataAddS16(addresidimfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-uri", 0, "define resid file URI", NULL); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-recip", 0, "define recipe", NULL); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg_skewness", 0, "define exposure background skewness", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg_kurtosis", 0, "define exposure background kurtosis", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bin_stdev", 0, "define exposure background binned stdev", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_0", 0, "define fringe slope (0th term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_1", 0, "define fringe slope (1st term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_2", 0, "define fringe slope (2nd term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_resid_0", 0, "define fringe residual (0th term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_resid_1", 0, "define fringe residual (1st term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_resid_2", 0, "define fringe residual (2nd term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + + // -residimfile + psMetadata *residimfileArgs = psMetadataAlloc(); + psMetadataAddS64(residimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddS32(residimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search for iteration number", 0); + psMetadataAddS64(residimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by detrend ID", 0); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID", NULL); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-recip", 0, "search for recipe", NULL); + psMetadataAddU64(residimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-select_state", 0, "search for state", NULL); + + // -revertresidimfile + psMetadata *revertresidimfileArgs = psMetadataAlloc(); + psMetadataAddS64(revertresidimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(revertresidimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddS64(revertresidimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by detrend ID", 0); + psMetadataAddStr(revertresidimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID", NULL); + psMetadataAddS16(revertresidimfileArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updateresidimfile + psMetadata *updateresidimfileArgs = psMetadataAlloc(); + psMetadataAddS64(updateresidimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(updateresidimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddS64(updateresidimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + + // -pendingcleanup_residimfile + psMetadata *pendingcleanup_residimfileArgs = psMetadataAlloc(); + psMetadataAddBool(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddS64(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + + // -donecleanup_residimfile + psMetadata *donecleanup_residimfileArgs = psMetadataAlloc(); + psMetadataAddBool(donecleanup_residimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanup_residimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(donecleanup_residimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS32(donecleanup_residimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddS64(donecleanup_residimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(donecleanup_residimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); + + // -toresidexp + psMetadata *toresidexpArgs = psMetadataAlloc(); + psMetadataAddU64(toresidexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(toresidexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addresidexp + psMetadata *addresidexpArgs = psMetadataAlloc(); + psMetadataAddS64(addresidexpArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddS32(addresidexpArgs, PS_LIST_TAIL, "-iteration", 0, "define iteration number", 0); + psMetadataAddS64(addresidexpArgs, PS_LIST_TAIL, "-exp_id", 0, "define detrend ID (required)", 0); + psMetadataAddS16(addresidexpArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-recip", 0, "define recipe", NULL); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg_skewness", 0, "define exposure background skewness", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg_kurtosis", 0, "define exposure background kurtosis", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bin_stdev", 0, "define exposure background binned stdev", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_0", 0, "define fringe slope (0th term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_1", 0, "define fringe slope (1st term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_2", 0, "define fringe slope (2nd term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_resid_0", 0, "define fringe residual (0th term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_resid_1", 0, "define fringe residual (1st term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_resid_2", 0, "define fringe residual (2nd term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddBool(addresidexpArgs, PS_LIST_TAIL, "-reject", 0, "exposure is not to be stacked in the next iteration", false); + + // -residexp + psMetadata *residexpArgs = psMetadataAlloc(); + psMetadataAddS64(residexpArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddS32(residexpArgs, PS_LIST_TAIL, "-iteration", 0, "search for iteration number", 0); + psMetadataAddS64(residexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search for exp ID", 0); + psMetadataAddStr(residexpArgs, PS_LIST_TAIL, "-recip", 0, "search for recipe", NULL); + psMetadataAddBool(residexpArgs, PS_LIST_TAIL, "-reject", 0, "search for acceptable residuals", false); + psMetadataAddU64(residexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(residexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(residexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertresidexp + psMetadata *revertresidexpArgs = psMetadataAlloc(); + psMetadataAddS64(revertresidexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(revertresidexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddS64(revertresidexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by detrend ID", 0); + psMetadataAddS16(revertresidexpArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updateresidexp + psMetadata *updateresidexpArgs = psMetadataAlloc(); + psMetadataAddS64(updateresidexpArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID", 0); + psMetadataAddS32(updateresidexpArgs, PS_LIST_TAIL, "-iteration", 0, "define iteration number", 0); + psMetadataAddS64(updateresidexpArgs, PS_LIST_TAIL, "-exp_id", 0, "define exp ID", 0); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-recip", 0, "define recipe", NULL); + psMetadataAddF64(updateresidexpArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(updateresidexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(updateresidexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddBool(updateresidexpArgs, PS_LIST_TAIL, "-reject", 0, "exposure is not to be stacked in the next iteration", false); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + + // -pendingcleanup_residexp + psMetadata *pendingcleanup_residexpArgs = psMetadataAlloc(); + psMetadataAddBool(pendingcleanup_residexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanup_residexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(pendingcleanup_residexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(pendingcleanup_residexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddS64(pendingcleanup_residexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by detrend ID", 0); + + // -donecleanup_residexp + psMetadata *donecleanup_residexpArgs = psMetadataAlloc(); + psMetadataAddBool(donecleanup_residexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanup_residexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(donecleanup_residexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(donecleanup_residexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddS64(donecleanup_residexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by detrend ID", 0); + + // -todetrunsummary + psMetadata *todetrunsummaryArgs = psMetadataAlloc(); + psMetadataAddBool(todetrunsummaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(todetrunsummaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -adddetrunsummary + psMetadata *adddetrunsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(adddetrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddS32(adddetrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, "define iteration number", 0); + psMetadataAddF64(adddetrunsummaryArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(adddetrunsummaryArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(adddetrunsummaryArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddS16(adddetrunsummaryArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddBool(adddetrunsummaryArgs, PS_LIST_TAIL, "-accept", 0, "declare that this detrun iteration is accepted as a master", false); + psMetadataAddBool(adddetrunsummaryArgs, PS_LIST_TAIL, "-again", 0, "start a new iteration of this detrend run", false); + + // -detrunsummary + psMetadata *detrunsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(detrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddS32(detrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, "search for iteration number", 0); + psMetadataAddU64(detrunsummaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(detrunsummaryArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(detrunsummaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(detrunsummaryArgs, PS_LIST_TAIL, "-reject", 0, "search for acceptable residuals", false); + + // -revertdetrunsummary + psMetadata *revertdetrunsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(revertdetrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(revertdetrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddS16(revertdetrunsummaryArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updatedetrunsummary + psMetadata *updatedetrunsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(updatedetrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend master for detrend ID (required)", 0); + psMetadataAddBool(updatedetrunsummaryArgs, PS_LIST_TAIL, "-accept", 0, "declare that this detrun iteration is accepted as a master", false); + psMetadataAddBool(updatedetrunsummaryArgs, PS_LIST_TAIL, "-reject", 0, "reject this detrun iteration as a master", false); + + // -updatedetrun + psMetadata *updatedetrunArgs = psMetadataAlloc(); + psMetadataAddS64(updatedetrunArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend master for detrend ID (required)", 0); + psMetadataAddBool(updatedetrunArgs, PS_LIST_TAIL, "-again", 0, "start a new iteration of this detrend run", false); + psMetadataAddStr(updatedetrunArgs, PS_LIST_TAIL, "-state", 0, "set the state of this detrend run", false); + + // -rerun + psMetadata *rerunArgs = psMetadataAlloc(); + psMetadataAddS64(rerunArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend master for detrend ID (required)", 0); + psMetadataAddS64(rerunArgs, PS_LIST_TAIL, "-exp_id", PS_META_DUPLICATE_OK, "include this exposure (multiple OK, required)", 0); + + // -register_detrend + psMetadata *register_detrendArgs = psMetadataAlloc(); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-det_type", 0, "define the type of detrend run (required)", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-filelevel", 0, "define filelevel (required)", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-inst", 0, "define camera", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-exp_type", 0, "define exposure type", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-filter", 0, "define filter ", NULL); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min exposure time", NAN); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max exposure time", NAN); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddTime(register_detrendArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddTime(register_detrendArgs, PS_LIST_TAIL, "-time_begin", 0, "detrend applies to exposures taken during this period", NULL); + psMetadataAddTime(register_detrendArgs, PS_LIST_TAIL, "-time_end", 0, "detrend applies to exposures taken during this period", NULL); + psMetadataAddTime(register_detrendArgs, PS_LIST_TAIL, "-use_begin", 0, "start of detrend run applicable period", NULL); + psMetadataAddTime(register_detrendArgs, PS_LIST_TAIL, "-use_end", 0, "end of detrend run applicable period", NULL); + psMetadataAddS64(register_detrendArgs, PS_LIST_TAIL, "-ref_det_id", 0, "define reference det_id", 0); + psMetadataAddS32(register_detrendArgs, PS_LIST_TAIL, "-ref_iter", 0, "define reference iteration", -1); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-label", 0, "define detrun label", NULL); + psMetadataAddBool(register_detrendArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -register_detrend_imfile + psMetadata *register_detrend_imfileArgs = psMetadataAlloc(); + psMetadataAddS64(register_detrend_imfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddStr(register_detrend_imfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID (required)", NULL); + psMetadataAddStr(register_detrend_imfileArgs, PS_LIST_TAIL, "-uri", 0, "define resid file URI (required)", NULL); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + psMetadataAddStr(register_detrend_imfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-pending", "list active detruns", DETTOOL_MODE_PENDING, pendingArgs); + PXOPT_ADD_MODE("-definebytag", "", DETTOOL_MODE_DEFINEBYTAG, definebytagArgs); + PXOPT_ADD_MODE("-definebyquery", "", DETTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definebydetrun", "", DETTOOL_MODE_DEFINEBYDETRUN, definebydetrunArgs); + PXOPT_ADD_MODE("-makecorrection", "", DETTOOL_MODE_MAKECORRECTION, makecorrectionArgs); + PXOPT_ADD_MODE("-tocorrectexp", "", DETTOOL_MODE_TOCORRECTEXP, tocorrectexpArgs); + PXOPT_ADD_MODE("-tocorrectimfile", "", DETTOOL_MODE_TOCORRECTIMFILE, tocorrectimfileArgs); + PXOPT_ADD_MODE("-addcorrectimfile", "", DETTOOL_MODE_ADDCORRECTIMFILE, addcorrectimfileArgs); + PXOPT_ADD_MODE("-raw", "", DETTOOL_MODE_RAW, rawArgs); + PXOPT_ADD_MODE("-runs", "", DETTOOL_MODE_RUNS, runsArgs); + PXOPT_ADD_MODE("-childlessrun", "", DETTOOL_MODE_CHILDLESSRUN, childlessrunArgs); + PXOPT_ADD_MODE("-input", "", DETTOOL_MODE_INPUT, inputArgs); + + PXOPT_ADD_MODE("-toprocessedimfile", "", DETTOOL_MODE_TOPROCESSEDIMFILE, toprocessedimfileArgs); + PXOPT_ADD_MODE("-addprocessedimfile", "", DETTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); + PXOPT_ADD_MODE("-processedimfile", "", DETTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); + PXOPT_ADD_MODE("-revertprocessedimfile", "", DETTOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileArgs); + PXOPT_ADD_MODE("-updateprocessedimfile", "", DETTOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileArgs); + PXOPT_ADD_MODE("-pendingcleanup_processedimfile", "", DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDIMFILE, pendingcleanup_processedimfileArgs); + PXOPT_ADD_MODE("-donecleanup_processedimfile", "", DETTOOL_MODE_DONECLEANUP_PROCESSEDIMFILE, donecleanup_processedimfileArgs); + + PXOPT_ADD_MODE("-toprocessedexp", "", DETTOOL_MODE_TOPROCESSEDEXP, toprocessedexpArgs); + PXOPT_ADD_MODE("-addprocessedexp", "", DETTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs); + PXOPT_ADD_MODE("-processedexp", "", DETTOOL_MODE_PROCESSEDEXP, processedexpArgs); + PXOPT_ADD_MODE("-revertprocessedexp", "", DETTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpArgs); + PXOPT_ADD_MODE("-updateprocessedexp", "", DETTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpArgs); + PXOPT_ADD_MODE("-pendingcleanup_processedexp", "", DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDEXP, pendingcleanup_processedexpArgs); + PXOPT_ADD_MODE("-donecleanup_processedexp", "", DETTOOL_MODE_DONECLEANUP_PROCESSEDEXP, donecleanup_processedexpArgs); + + PXOPT_ADD_MODE("-tostacked", "", DETTOOL_MODE_TOSTACKED, tostackedArgs); + PXOPT_ADD_MODE("-addstacked", "", DETTOOL_MODE_ADDSTACKED, addstackedArgs); + PXOPT_ADD_MODE("-stacked", "", DETTOOL_MODE_STACKED, stackedArgs); + PXOPT_ADD_MODE("-revertstacked", "", DETTOOL_MODE_REVERTSTACKED, revertstackedArgs); + PXOPT_ADD_MODE("-updatestacked", "", DETTOOL_MODE_UPDATESTACKED, updatestackedArgs); + PXOPT_ADD_MODE("-pendingcleanup_stacked", "", DETTOOL_MODE_PENDINGCLEANUP_STACKED, pendingcleanup_stackedArgs); + PXOPT_ADD_MODE("-donecleanup_stacked", "", DETTOOL_MODE_DONECLEANUP_STACKED, donecleanup_stackedArgs); + + PXOPT_ADD_MODE("-tonormalizedstat", "", DETTOOL_MODE_TONORMALIZEDSTAT, tonormalizedstatArgs); + PXOPT_ADD_MODE("-addnormalizedstat", "", DETTOOL_MODE_ADDNORMALIZEDSTAT, addnormstatArgs); + PXOPT_ADD_MODE("-normalizedstat", "", DETTOOL_MODE_NORMALIZEDSTAT, normalizedstatArgs); + PXOPT_ADD_MODE("-revertnormalizedstat", "", DETTOOL_MODE_REVERTNORMALIZEDSTAT, revertnormalizedstatArgs); + PXOPT_ADD_MODE("-updatenormalizedstat", "", DETTOOL_MODE_UPDATENORMALIZEDSTAT, updatenormalizedstatArgs); + PXOPT_ADD_MODE("-pendingcleanup_normalizedstat", "", DETTOOL_MODE_PENDINGCLEANUP_NORMALIZEDSTAT, pendingcleanup_normalizedstatArgs); + PXOPT_ADD_MODE("-donecleanup_normalizedstat", "", DETTOOL_MODE_DONECLEANUP_NORMALIZEDSTAT, donecleanup_normalizedstatArgs); + + PXOPT_ADD_MODE("-tonormalize", "", DETTOOL_MODE_TONORMALIZE, tonormalizeArgs); + PXOPT_ADD_MODE("-addnormalizedimfile", "", DETTOOL_MODE_ADDNORMALIZEDIMFILE,addnormalizedimfileArgs); + PXOPT_ADD_MODE("-normalizedimfile","", DETTOOL_MODE_NORMALIZEDIMFILE, normalizedimfileArgs); + PXOPT_ADD_MODE("-revertnormalizedimfile","", DETTOOL_MODE_REVERTNORMALIZEDIMFILE, revertnormalizedimfileArgs); + PXOPT_ADD_MODE("-updatenormalizedimfile","", DETTOOL_MODE_UPDATENORMALIZEDIMFILE, updatenormalizedimfileArgs); + PXOPT_ADD_MODE("-pendingcleanup_normalizedimfile", "", DETTOOL_MODE_PENDINGCLEANUP_NORMALIZEDIMFILE, pendingcleanup_normalizedimfileArgs); + PXOPT_ADD_MODE("-donecleanup_normalizedimfile", "", DETTOOL_MODE_DONECLEANUP_NORMALIZEDIMFILE, donecleanup_normalizedimfileArgs); + + PXOPT_ADD_MODE("-tonormalizedexp", "", DETTOOL_MODE_TONORMALIZEDEXP, tonormalizedexpArgs); + PXOPT_ADD_MODE("-addnormalizedexp", "", DETTOOL_MODE_ADDNORMALIZEDEXP, addnormalizedexpArgs); + PXOPT_ADD_MODE("-normalizedexp", "", DETTOOL_MODE_NORMALIZEDEXP, normalizedexpArgs); + PXOPT_ADD_MODE("-revertnormalizedexp","", DETTOOL_MODE_REVERTNORMALIZEDEXP, revertnormalizedexpArgs); + PXOPT_ADD_MODE("-updatenormalizedexp","", DETTOOL_MODE_UPDATENORMALIZEDEXP, updatenormalizedexpArgs); + PXOPT_ADD_MODE("-pendingcleanup_normalizedexp", "", DETTOOL_MODE_PENDINGCLEANUP_NORMALIZEDEXP, pendingcleanup_normalizedexpArgs); + PXOPT_ADD_MODE("-donecleanup_normalizedexp", "", DETTOOL_MODE_DONECLEANUP_NORMALIZEDEXP, donecleanup_normalizedexpArgs); + + PXOPT_ADD_MODE("-toresidimfile", "", DETTOOL_MODE_TORESIDIMFILE, toresidimfileArgs); + PXOPT_ADD_MODE("-addresidimfile", "", DETTOOL_MODE_ADDRESIDIMFILE, addresidimfileArgs); + PXOPT_ADD_MODE("-residimfile", "", DETTOOL_MODE_RESIDIMFILE, residimfileArgs); + PXOPT_ADD_MODE("-revertresidimfile", "", DETTOOL_MODE_REVERTRESIDIMFILE, revertresidimfileArgs); + PXOPT_ADD_MODE("-updateresidimfile", "", DETTOOL_MODE_UPDATERESIDIMFILE, updateresidimfileArgs); + PXOPT_ADD_MODE("-pendingcleanup_residimfile", "", DETTOOL_MODE_PENDINGCLEANUP_RESIDIMFILE, pendingcleanup_residimfileArgs); + PXOPT_ADD_MODE("-donecleanup_residimfile", "", DETTOOL_MODE_DONECLEANUP_RESIDIMFILE, donecleanup_residimfileArgs); + + PXOPT_ADD_MODE("-toresidexp", "", DETTOOL_MODE_TORESIDEXP, toresidexpArgs); + PXOPT_ADD_MODE("-addresidexp", "", DETTOOL_MODE_ADDRESIDEXP, addresidexpArgs); + PXOPT_ADD_MODE("-residexp", "", DETTOOL_MODE_RESIDEXP, residexpArgs); + PXOPT_ADD_MODE("-revertresidexp", "", DETTOOL_MODE_REVERTRESIDEXP, revertresidexpArgs); + PXOPT_ADD_MODE("-updateresidexp", "", DETTOOL_MODE_UPDATERESIDEXP, updateresidexpArgs); + PXOPT_ADD_MODE("-pendingcleanup_residexp", "", DETTOOL_MODE_PENDINGCLEANUP_RESIDEXP, pendingcleanup_residexpArgs); + PXOPT_ADD_MODE("-donecleanup_residexp", "", DETTOOL_MODE_DONECLEANUP_RESIDEXP, donecleanup_residexpArgs); + + PXOPT_ADD_MODE("-todetrunsummary", "", DETTOOL_MODE_TODETRUNSUMMARY, todetrunsummaryArgs); + PXOPT_ADD_MODE("-adddetrunsummary", "", DETTOOL_MODE_ADDDETRUNSUMMARY,adddetrunsummaryArgs); + PXOPT_ADD_MODE("-detrunsummary", "", DETTOOL_MODE_DETRUNSUMMARY,detrunsummaryArgs); + PXOPT_ADD_MODE("-revertdetrunsummary", "", DETTOOL_MODE_REVERTDETRUNSUMMARY, revertdetrunsummaryArgs); + PXOPT_ADD_MODE("-updatedetrunsummary", "", DETTOOL_MODE_UPDATEDETRUNSUMMARY, updatedetrunsummaryArgs); + + PXOPT_ADD_MODE("-updatedetrun", "", DETTOOL_MODE_UPDATEDETRUN, updatedetrunArgs); + PXOPT_ADD_MODE("-rerun", "", DETTOOL_MODE_RERUN, rerunArgs); + PXOPT_ADD_MODE("-register_detrend", "", DETTOOL_MODE_REGISTER_DETREND, register_detrendArgs); + PXOPT_ADD_MODE("-register_detrend_imfile", "", DETTOOL_MODE_REGISTER_DETREND_IMFILE, register_detrend_imfileArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_correction.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_correction.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_correction.c (revision 22161) @@ -0,0 +1,254 @@ +/* + * dettool_correction.c + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +bool makecorrectionMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(ref_det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_S32(ref_iter, config->args, "-iteration", true, false); // required + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); // optional + + // build the needed where + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", ref_det_id); + psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", ref_iter); + psArray *runs = detRunSelectRowObjects(config->dbh, where, 1); + psFree (where); + + if (!psArrayLength(runs)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + detRunRow *detRun = runs->data[0]; + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + detRunInsert(config->dbh, + 0, // det_id + 0, // iteration + detRun->det_type, + "correction", // mode + "run", // state + detRun->filelevel, + detRun->workdir, + detRun->camera, + detRun->telescope, + detRun->exp_type, + detRun->reduction, + detRun->filter, + detRun->airmass_min, + detRun->airmass_max, + detRun->exp_time_min, + detRun->exp_time_max, + detRun->ccd_temp_min, + detRun->ccd_temp_max, + detRun->posang_min, + detRun->posang_max, + detRun->registered, + detRun->time_begin, + detRun->time_end, + detRun->use_begin, + detRun->use_end, + detRun->solang_min, + detRun->solang_max, + detRun->label, + detRun->det_id, // ref_det_id + detRun->iteration // ref_iter + ); + + psFree(runs); + + // print the new detRun + psS64 new_det_id = psDBLastInsertID(config->dbh); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psMetadata *where_new = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", new_det_id); + psArray *detRuns = psDBSelectRows(config->dbh, "detRun", where_new, 0); + psFree(where_new); + + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check results + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 "(this should not happen)", new_det_id); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, detRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detRuns); + return false; + } + psFree(detRuns); + + return true; +} + +bool tocorrectexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_tocorrectexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool tocorrectimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_type", "det_type", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_tocorrectimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + // NOTE the SQL uses an intermediate table 'det1' for this query + psString whereClause = psDBGenerateWhereConditionSQL(where, "det1"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree (where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingCorrectImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +// NOTE : the command-line args are parsed by register_detrend_imfileMode +bool addcorrectimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // insert the row into detRegisterImfile + if (!register_detrend_imfileMode(config)) { + return false; + } + + // automatically stop completed 'correct' detRuns + psString query = pxDataGet("dettool_stop_completed_correct_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_detrunsummary.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_detrunsummary.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_detrunsummary.c (revision 22161) @@ -0,0 +1,366 @@ +/* + * dettool_detrunsummary.c + * + * Copyright (C) 2006 Joshua Hoblitt & EAM; IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +static detRunSummaryRow *mdToDetRunSummary(pxConfig *config, psMetadata *row); + +/* The SQL returns a list of detrend runs (with detrend id, iteration and detrend type) which + * have completed all residexps. + */ + +bool todetrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_todetrunsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detRejectExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +// this function is used to generate the detrunSummary entries used below +// XXX why is this a separate function? roll it back into adddetrunsummary? +static detRunSummaryRow *mdToDetRunSummary(pxConfig *config, psMetadata *row) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + // from row + psS64 det_id = psMetadataLookupS64(&status, row, "det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for det_id"); + return false; + } + psS32 iteration = psMetadataLookupS32(&status, row, "iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for iteration"); + return false; + } + + // optional + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + PXOPT_LOOKUP_BOOL(accept, config->args, "-accept", false); + + return detRunSummaryRowAlloc( + det_id, + iteration, + "full", + bg, + bg_stdev, + bg_mean_stdev, + accept, + code + ); +} + +// XXX data_state should be set to 'full' here +bool adddetrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // build a query to search by det_id, iteration, exp_id + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_BOOL(again, config->args, "-again", false); + + // The values supplied as arguments on the command (eg, -bg) are parsed + // by mdToDetRunSummary below. + // XXX why is there ever more than one? + + psString query = pxDataGet("dettool_find_completed_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // XXX why is there ever more than one result here? + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + // convert metadata into a detResidExp object + detRunSummaryRow *runSummary = mdToDetRunSummary(config, row); + if (!runSummary) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata to detResidExp"); + psFree(output); + return false; + } + // insert detResidExp object into the database + if (!detRunSummaryInsertObject(config->dbh, runSummary)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(runSummary); + psFree(output); + return false; + } + psFree(runSummary); + } + + psFree(output); + + // XXX this logic does not deal with the case of -code being set + // XXX it should be an error for -again and -code to both be set + if (again) { + if (!startNewIteration(config, det_id)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to start new iteration"); + return false; + } + } else { + // set detRun.state to stop + if (!setDetRunState(config, det_id, "stop")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set detRun.state"); + return false; + } + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; +} + +bool detrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psString query = pxDataGet("dettool_detrunsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "NULL"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detRunSummary.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detRunSummary.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawDetrendImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool revertdetrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertdetrunsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detRunSummary"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +// XXX need to add -data_state here +bool updatedetrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_BOOL(accept, config->args, "-accept", false); + PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false); + + if (accept && reject) { + psError(PS_ERR_UNKNOWN, true, "-accept and -reject are exclusive"); + return false; + } + + if (!(accept || reject)) { + psError(PS_ERR_UNKNOWN, true, "either -accept or -reject is required"); + return false; + } + + char *query = "UPDATE detRunSummary SET accept = %d WHERE det_id = %"PRId64; + if (!p_psDBRunQuery(config->dbh, query, accept, det_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_normalizedexp.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_normalizedexp.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_normalizedexp.c (revision 22161) @@ -0,0 +1,389 @@ +/* + * dettool_normalizedexp.c + * + * Copyright (C) 2006 Joshua Hoblitt & EAM; IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +bool tonormalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_tonormalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingNormExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} +bool addnormalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); // Required if code == 0 + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + psString query = pxDataGet("dettool_tonormalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX in some other places, we put the arguments in the .sql and supply them to RunQuery + psStringAppend(&query, " WHERE det_id = %" PRId64 " AND iteration = %d", det_id, iteration); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + psFree(output); + + // insert the new row into the detProcessedImfile table + if (!detNormalizedExpInsert( + config->dbh, + det_id, + iteration, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + "full", + code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +bool normalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-recip", "recipe", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psString query = pxDataGet("dettool_normalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detNormalizedExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detNormalizedExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detNormalizedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + + +bool revertnormalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertnormalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detNormalizedExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +bool updatenormalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", true, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); + + if (!setNormExpDataState(config, det_id, iteration, data_state)) { + return false; + } + return true; +} + +bool pendingcleanup_normalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detNormalizedExp.det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "detNormalizedExp.iteration", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_normalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detCleanupNormExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool donecleanup_normalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detNormalizedExp.det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "detNormalizedExp.iteration", "=="); + + psString query = pxDataGet("dettool_donecleanup_normalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detDoneCleanup_normalizedexp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_normalizedimfile.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_normalizedimfile.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_normalizedimfile.c (revision 22161) @@ -0,0 +1,372 @@ +/* + * dettool_normalizedimfile.c + * + * Copyright (C) 2006 Joshua Hoblitt & EAM; IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +// XXX this one is inconsistent - rename 'tonormalizedimfile' and change ipp_serial_detrend.pl and detrend.norm.pro +bool tonormalizeMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_tonormalize.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingNormImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool addnormalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + // Required if code == 0 + PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false); + + // optional + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + if (!detNormalizedImfileInsert( + config->dbh, + det_id, + iteration, + class_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + "full", + code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +bool normalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-recip", "recipe", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psString query = pxDataGet("dettool_normalizedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detNormalizedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detNormalizedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detNormalizedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detNormalizedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool revertnormalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertnormalizedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detNormalizedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +bool updatenormalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); + + if (!setNormImfileDataState(config, det_id, iteration, class_id, data_state)) { + return false; + } + return true; +} + +bool pendingcleanup_normalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detNormalizedImfile.det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "detNormalizedImfile.iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "detNormalizedImfile.det_id", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_normalizedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detCleanupNormImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool donecleanup_normalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detNormalizedImfile.det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "detNormalizedImfile.iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "detNormalizedImfile.det_id", "=="); + + psString query = pxDataGet("dettool_donecleanup_normalizedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detDoneCleanup_normalizedimfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_normalizedstat.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_normalizedstat.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_normalizedstat.c (revision 22161) @@ -0,0 +1,352 @@ +/* + * dettool_normalizedstat.c + * + * Copyright (C) 2006 Joshua Hoblitt & EAM; IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +bool tonormalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_tonormalizedstat.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingNormStatImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool addnormalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // required + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + // optional + PXOPT_LOOKUP_F32(norm, config->args, "-norm", false, false); + + // default + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + detNormalizedStatImfileRow *row = detNormalizedStatImfileRowAlloc + (det_id, + iteration, + class_id, + norm, + "full", + code); + + if (!detNormalizedStatImfileInsertObject(config->dbh, row)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +bool normalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_normalizedstat.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "WHERE fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "WHERE fault = 0"); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detNormalizedStatImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool revertnormalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertnormalizedstat.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detNormalizedStatImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +bool updatenormalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); + + if (!setNormStatImfileDataState(config, det_id, iteration, class_id, data_state)) { + return false; + } + return true; +} + +bool pendingcleanup_normalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detNormalizedStatImfile.det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "detNormalizedStatImfile.iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "detNormalizedStatImfile.det_id", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_normalizedstat.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detCleanupNormStatImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool donecleanup_normalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detNormalizedStatImfile.det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "detNormalizedStatImfile.iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "detNormalizedStatImfile.det_id", "=="); + + psString query = pxDataGet("dettool_donecleanup_normalizedstat.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detDoneCleanup_normalizedstat", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_processedexp.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_processedexp.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_processedexp.c (revision 22161) @@ -0,0 +1,398 @@ +/* + * dettool_processedexp.c + * + * Copyright (C) 2006 Joshua Hoblitt & EAM; IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +bool toprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_toprocessedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool addprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id, exp_id, recip, -bg, -bg_stdev + // are required + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + // Required if code == 0 + PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); + + // optional + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(fringe_0, config->args, "-fringe_0", false, false); + PXOPT_LOOKUP_F64(fringe_1, config->args, "-fringe_1", false, false); + PXOPT_LOOKUP_F64(fringe_2, config->args, "-fringe_2", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + + psString query = pxDataGet("dettool_addprocessedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query, (long long) det_id, (long long) exp_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + psFree(output); + + // create a new detProcessedImfile object + detProcessedExpRow *detRow = detProcessedExpRowAlloc( + det_id, + exp_id, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + fringe_0, + fringe_1, + fringe_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + "full", + code + ); + + // insert the new row into the detProcessedImfile table + if (!detProcessedExpInsertObject(config->dbh, detRow)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detRow); + return false; + } + + psFree(detRow); + + return true; +} + +bool processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM detProcessedExp"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detProcessedExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detProcessedExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool revertprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertprocessedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detProcessedExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +bool updateprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); + if (!setProcessedExpDataState(config, det_id, exp_id, data_state)) { + return false; + } + return true; +} + +bool pendingcleanup_processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_processedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detCleanupProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +// XXX SQL missing +bool donecleanup_processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + psString query = pxDataGet("dettool_donecleanup_processedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detDoneCleanupProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_processedimfile.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_processedimfile.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_processedimfile.c (revision 22161) @@ -0,0 +1,433 @@ +/* + * dettool_processedimfile.c + * + * Copyright (C) 2006 Joshua Hoblitt & EAM; IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +bool toprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_toprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree (where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingProcessedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool addprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id, exp_id, class_id, uri, recipe, -bg, -bg_stdev are required + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + // Required if code == 0 + PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false); + PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); + + // optional + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(fringe_0, config->args, "-fringe_0", false, false); + PXOPT_LOOKUP_F64(fringe_1, config->args, "-fringe_1", false, false); + PXOPT_LOOKUP_F64(fringe_2, config->args, "-fringe_2", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + // find the matching rawImfile by exp_id/class_id + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psArray *rawImfiles = rawImfileSelectRowObjects(config->dbh, where, 0); + psFree(where); + + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found "); + return false; + } + + // create a new detProcessedImfile object + detProcessedImfileRow *detRow = detProcessedImfileRowAlloc( + det_id, + exp_id, + class_id, + uri, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + fringe_0, + fringe_1, + fringe_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + "full", + code + ); + psFree(rawImfiles); + + // insert the new row into the detProcessedImfile table + if (!detProcessedImfileInsertObject(config->dbh, detRow)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detRow); + return false; + } + + psFree(detRow); + + return true; +} + +bool processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool hasWhere = false; + + PXOPT_LOOKUP_BOOL(included, config->args, "-included", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detProcessedImfile.det_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "detProcessedImfile.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "detProcessedImfile.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-select_state", "detRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-select_mode", "detRun.mode", "=="); + + psString query = pxDataGet("dettool_processedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + hasWhere = true; + } + psFree (where); + + // restrict search to included imfiles + if (included) { + if (hasWhere) { + psStringAppend(&query, " AND detInputExp.include = 1"); + } else { + psStringAppend(&query, " WHERE detInputExp.include = 1"); + } + hasWhere = true; + } + + if (hasWhere) { + psStringAppend(&query, " AND"); + } else { + psStringAppend(&query, " WHERE"); + } + + if (faulted) { + psStringAppend(&query, " %s", " detProcessedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", " detProcessedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool revertprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detProcessedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +bool updateprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); + + if (!setProcessedImfileDataState(config, det_id, exp_id, class_id, data_state)) { + return false; + } + return true; +} + +bool pendingcleanup_processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_processedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detCleanupProcessedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +// XXX SQL missing +bool donecleanup_processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString query = pxDataGet("dettool_donecleanup_processedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detDoneCleanup_processedimfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_residexp.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_residexp.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_residexp.c (revision 22161) @@ -0,0 +1,479 @@ +/* + * dettool_residexp.c + * + * Copyright (C) 2006 Joshua Hoblitt & EAM; IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +/* + The SQL returns a list of exposures for which all component class ids have had residuals + created. This list includes the detrend id, iteration, exposure id, detrend type, and + whether the exposure was included in the stack for this iteration. +*/ + +bool toresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_toresidexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingResidExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +/* + The SQL returns a list of exposures for which all component class ids have had residuals + created. This list includes the detrend id, iteration, exposure id, detrend type, and + whether the exposure was included in the stack for this iteration. +*/ + +bool addresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // limit search by det_id, iteration, exp_id + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); // required + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); // Required if code == 0 + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(bg_skewness, config->args, "-bg_skewness", false, false); + PXOPT_LOOKUP_F64(bg_kurtosis, config->args, "-bg_kurtosis", false, false); + PXOPT_LOOKUP_F64(bin_stdev, config->args, "-bin_stdev", false, false); + PXOPT_LOOKUP_F64(fringe_0, config->args, "-fringe_0", false, false); + PXOPT_LOOKUP_F64(fringe_1, config->args, "-fringe_1", false, false); + PXOPT_LOOKUP_F64(fringe_2, config->args, "-fringe_2", false, false); + PXOPT_LOOKUP_F64(fringe_resid_0, config->args, "-fringe_resid_0", false, false); + PXOPT_LOOKUP_F64(fringe_resid_1, config->args, "-fringe_resid_1", false, false); + PXOPT_LOOKUP_F64(fringe_resid_2, config->args, "-fringe_resid_2", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false); + + psString query = pxDataGet("dettool_toresidexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no detResidImfile rows found"); + psFree(output); + return false; + } + psFree(output); + + if (!detResidExpInsert( + config->dbh, + det_id, + iteration, + exp_id, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + bg_skewness, + bg_kurtosis, + bin_stdev, + fringe_0, + fringe_1, + fringe_2, + fringe_resid_0, + fringe_resid_1, + fringe_resid_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + "full", + !reject, + code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +bool residexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-recip", "recipe", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false); + + psString query = pxDataGet("dettool_residexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detResidExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detResidExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detResidExp.fault = 0"); + } + + // list only accepted rows + // XXX the usage of this flag is unclear : -reject means "accepted"? + if (reject) { + psStringAppend(&query, " %s", "AND detResidExp.accept != 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detResidExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool revertresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertresidexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detResidExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +bool updateresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false); + + // build a query to search by det_id, iteration, exp_id + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + // find the values we're going to set + // copy everything but det_id, iteration, & exp_id from the args and + // remove the '-' prefix + psMetadata *set = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, set, "-recip", "recipe", "=="); + PXOPT_COPY_F64(config->args, set, "-bg", "bg", "=="); + PXOPT_COPY_F64(config->args, set, "-bg_stdev", "bg_stdev", "=="); + PXOPT_COPY_F64(config->args, set, "-bg_mean_stdev", "bg_mean_stdev", "=="); + PXOPT_COPY_STR(config->args, set, "-path_base", "path_base", "=="); + PXOPT_COPY_STR(config->args, set, "-data_state", "data_state", "=="); + + // double-check the allowed data_state values + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", false, false); + if (data_state) { + if (!isValidDataState (data_state)) return false; + } + + // this can't be PXOPT_ macro-ized as reject is !'d + if (!psMetadataAddBool(set, PS_LIST_TAIL, "accept", 0, "==", !reject)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item accept"); + psFree(set); + psFree(where); + return false; + } + + long changed = psDBUpdateRows(config->dbh, "detResidExp", where, set); + psFree(set); + psFree(where); + + if (changed < 0) { + psError(PS_ERR_UNKNOWN, false, "no rows were updated"); + return false; + } + + return true; +} + +bool pendingcleanup_residexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_residexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detCleanupResidExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool donecleanup_residexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + psString query = pxDataGet("dettool_donecleanup_residexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detDoneCleanup_residexp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_residimfile.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_residimfile.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_residimfile.c (revision 22161) @@ -0,0 +1,410 @@ +/* + * dettool_residimfile.c + * + * Copyright (C) 2006 Joshua Hoblitt & EAM; IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +bool toresidimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_toresidimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingResidImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool addresidimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); + + PXOPT_LOOKUP_S64(ref_det_id, config->args, "-ref_det_id", true, false); // required + PXOPT_LOOKUP_S32(ref_iter, config->args, "-ref_iter", true, false); + + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); // required + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); // required + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false); // Required if code == 0 + PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); // Required if code == 0 + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(bg_skewness, config->args, "-bg_skewness", false, false); + PXOPT_LOOKUP_F64(bg_kurtosis, config->args, "-bg_kurtosis", false, false); + PXOPT_LOOKUP_F64(bin_stdev, config->args, "-bin_stdev", false, false); + PXOPT_LOOKUP_F64(fringe_0, config->args, "-fringe_0", false, false); + PXOPT_LOOKUP_F64(fringe_1, config->args, "-fringe_1", false, false); + PXOPT_LOOKUP_F64(fringe_2, config->args, "-fringe_2", false, false); + PXOPT_LOOKUP_F64(fringe_resid_0, config->args, "-fringe_resid_0", false, false); + PXOPT_LOOKUP_F64(fringe_resid_1, config->args, "-fringe_resid_1", false, false); + PXOPT_LOOKUP_F64(fringe_resid_2, config->args, "-fringe_resid_2", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + if (!detResidImfileInsert( + config->dbh, + det_id, + iteration, + ref_det_id, + ref_iter, + exp_id, + class_id, + uri, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + bg_skewness, + bg_kurtosis, + bin_stdev, + fringe_0, + fringe_1, + fringe_2, + fringe_resid_0, + fringe_resid_1, + fringe_resid_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + "full", + code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +bool residimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-recip", "recipe", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psString query = pxDataGet("dettool_residimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // NOTE: the query ends in a WHERE, add the required restriction on detRun.state + char *value = NULL; + bool status; + if ((value = psMetadataLookupStr(&status, config->args, "-select_state"))) { + psStringAppend(&query, " detRun.state = '%s'", value); + } else { + psStringAppend(&query, " detRun.state = 'run'"); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detResidImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detResidImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detResidImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawResidImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool revertresidimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertresidimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detResidImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +bool updateresidimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); + if (!setResidImfileDataState(config, det_id, iteration, exp_id, class_id, data_state)) { + return false; + } + return true; +} + +bool pendingcleanup_residimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_residimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detCleanupResidImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool donecleanup_residimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString query = pxDataGet("dettool_donecleanup_residimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detDoneCleanup_residimfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_stack.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_stack.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/dettool_stack.c (revision 22161) @@ -0,0 +1,457 @@ +/* + * dettool_stack.c + * + * Copyright (C) 2006 Joshua Hoblitt & EAM; IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dettool.h" + +static psArray *searchRawImfiles(pxConfig *config); + +bool tostackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_tostacked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingStackedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +// this is NOT a command-line mode : it is used by 'addstackedMode' +static psArray *searchRawImfiles(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + + // select exp_ids from detInputExp matching det_id & iteration + // where query should be pre-generated + psArray *detInputExp = detInputExpSelectRowObjects(config->dbh, where, 0); + psFree (where); + + if (!detInputExp) { + psError(PS_ERR_UNKNOWN, false, "no rawExp rows found"); + return NULL; + } + + // generate where query with just the exp_ids + psMetadata *where_exp_ids = psMetadataAlloc(); + for (long i = 0; i < psArrayLength(detInputExp); i++) { + detInputExpRow *row = detInputExp->data[i]; + if (!psMetadataAddS64(where_exp_ids, PS_LIST_TAIL, "exp_id", + PS_META_DUPLICATE_OK, "==", row->exp_id) + ) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(detInputExp); + psFree(where_exp_ids); + return NULL; + } + } + psFree(detInputExp); + + // select rawImfiles with matching exp_ids + psArray *rawImfiles = + rawImfileSelectRowObjects(config->dbh, where_exp_ids, 0); + psFree(where_exp_ids); + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found"); + return NULL; + } + + return rawImfiles; +} + +bool addstackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id, iteration, class_id, uri, & recipe are required + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + // Required if code == 0 + PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false); + PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); + + // optional + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + + // correlate the class_id against the input exposure(s) + // searchRawImfiles defines a where clause to search by det_id and iteration + psArray *rawImfiles = searchRawImfiles(config); + + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "failed to get rawImfiles from db"); + return false; + } + + bool valid_class_id = false; + if (rawImfiles) { + for (long i = 0; i < psArrayLength(rawImfiles); i++) { + if (!rawImfiles->data[i]) { + fprintf (stderr, "*"); + continue; + } + if (strcmp(class_id, ((rawImfileRow *)rawImfiles->data[i])->class_id) == 0) { + valid_class_id = true; + break; + } + } + psFree(rawImfiles); + } + + if (!valid_class_id) { + psError(PS_ERR_UNKNOWN, true, + "class_id can not be correlated with the input exposures"); + return false; + } + + // create a new detStackedImfile object + detStackedImfileRow *stackedImfile = detStackedImfileRowAlloc( + det_id, + iteration, + class_id, + uri, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + "full", + code + ); + + // insert the new row into the detProcessedImfile table + if (!detStackedImfileInsertObject(config->dbh, stackedImfile)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(stackedImfile); + return false; + } + + psFree(stackedImfile); + + return true; +} + +bool stackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-recip", "recipe", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("dettool_stacked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detStackedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detStackedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detStackedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +bool revertstackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertstacked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detStackedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + +bool updatestackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); + + if (!setStackedImfileDataState(config, det_id, iteration, class_id, data_state)) { + return false; + } + return true; +} + +bool pendingcleanup_stackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detStackedImfile.det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "detStackedImfile.iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "detStackedImfile.class_id", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_stacked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detCleanupStackedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +// XXX SQL missing +bool donecleanup_stackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "detStackedImfile.det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "detStackedImfile.iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "detStackedImfile.class_id", "=="); + + psString query = pxDataGet("dettool_donecleanup_stacked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detDoneCleanup_stacked", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/difftool.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/difftool.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/difftool.c (revision 22161) @@ -0,0 +1,1213 @@ +/* + * difftool.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "difftool.h" + +static bool definerunMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool addinputskyfileMode(pxConfig *config); +static bool inputskyfileMode(pxConfig *config); +static bool todiffskyfileMode(pxConfig *config); +static bool adddiffskyfileMode(pxConfig *config); +static bool diffskyfileMode(pxConfig *config); +static bool revertdiffskyfileMode(pxConfig *config); +static bool definepoprunMode(pxConfig *config); +static bool definebyqueryMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupskyfileMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); + +static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = difftoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(DIFFTOOL_MODE_DEFINERUN, definerunMode); + MODECASE(DIFFTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(DIFFTOOL_MODE_ADDINPUTSKYFILE, addinputskyfileMode); + MODECASE(DIFFTOOL_MODE_INPUTSKYFILE, inputskyfileMode); + MODECASE(DIFFTOOL_MODE_TODIFFSKYFILE, todiffskyfileMode); + MODECASE(DIFFTOOL_MODE_ADDDIFFSKYFILE, adddiffskyfileMode); + MODECASE(DIFFTOOL_MODE_DIFFSKYFILE, diffskyfileMode); + MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileMode); + MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunMode); + MODECASE(DIFFTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); + MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode); + MODECASE(DIFFTOOL_MODE_DONECLEANUP, donecleanupMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + + // default + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + diffRunRow *run = diffRunRowAlloc( + 0, // ID + "reg", // state + label, + reduction, + workdir, + NULL, // dvodb + registered, + skycell_id, + tess_id + ); + if (!run) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc diffRun object"); + return true; + } + if (!diffRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return true; + } + + // get the assigned diff_id + run->diff_id = psDBLastInsertID(config->dbh); + + if (!diffRunPrintObject(stdout, run, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(run); + return false; + } + + psFree(run); + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + + if (state) { + // set detRun.state to state + return setdiffRunState(config, diff_id, state); + } + + return true; +} + + +static bool addinputskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + + // optional + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false); + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", false, false); + PXOPT_LOOKUP_STR(kind, config->args, "-kind", false, false); + + // defaults to false + PXOPT_LOOKUP_BOOL(template, config->args, "-template", false); + + // must provide either stack_id or warp_id but not BOTH + if (!(stack_id || warp_id)) { + psError(PS_ERR_UNKNOWN, true, "either -stack_id or -warp_id must be specified"); + return false; + } + if (stack_id && warp_id) { + psError(PS_ERR_UNKNOWN, true, "either -stack_id or -warp_id must be specified"); + return false; + } + + // if a warp_id was provided we need to lookup the skycell_id and tess_id + // from the warpRun + psString skycell_id = NULL; + psString tess_id = NULL; + if (warp_id) { + if (!p_psDBRunQuery(config->dbh, "SELECT * from diffRun WHERE diff_id = %" PRId64, diff_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, false, "diff_id %" PRId64 " not found", diff_id); + psFree(output); + return false; + } + + diffRunRow *run = diffRunObjectFromMetadata(output->data[0]); + skycell_id = run->skycell_id; + tess_id = run->tess_id; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!diffInputSkyfileInsert(config->dbh, + diff_id, + template, + stack_id ? stack_id : PS_MAX_S64, // defined or NULL + warp_id ? warp_id : PS_MAX_S64, // defined or NULL + skycell_id, + tess_id, + kind + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, "SELECT count(diff_id) FROM diffRun JOIN diffInputSkyfile USING(diff_id) WHERE diff_id = %" PRId64, diff_id)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "diff_id %" PRId64 " not found", diff_id); + psFree(output); + return false; + } + + bool status; + psS32 count = psMetadataLookupS32(&status, output->data[0], "count(diff_id)"); + + if (count == 2) { + if (!setdiffRunState(config, diff_id, "new")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool inputskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-kind", "kind", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(template, config->args, "-template", false); + PXOPT_LOOKUP_BOOL(input, config->args, "-input", false); + + if (template && input) { + // User apparently wants both, which is the default behaviour + template = false; + input = false; + } + + // find all rawImfiles matching the default query + psString query = pxDataGet("difftool_inputskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = NULL; + if (psListLength(where->list)) { + whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringPrepend(&whereClause, "\n AND "); + } + psFree(where); + + // Add condition to get only templates or only inputs + { + psString templateClause = NULL; + if (template) { + psStringAppend(&templateClause, " %s", " template != 0"); + } else if (input) { + psStringAppend(&templateClause, " %s", " template = 0"); + } + if (templateClause) { + psStringAppend(&whereClause, "\n AND %s", templateClause); + } + psFree(templateClause); + } + + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query, whereClause, whereClause)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereClause); + psFree(query); + return false; + } + psFree(whereClause); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffInputSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool todiffskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "diffRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("difftool_todiffskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool adddiffskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); // required + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", (code == 0), false); + + // optional + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F32(dtime_diff, config->args, "-dtime_diff", false, false); + PXOPT_LOOKUP_S32(stamps_num, config->args, "-stamps_num", false, false); + PXOPT_LOOKUP_F32(stamps_rms, config->args, "-stamps_rms", false, false); + PXOPT_LOOKUP_S32(sources, config->args, "-sources", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!diffSkyfileInsert(config->dbh, + diff_id, + uri, + path_base, + "full", // data_state + bg, + bg_stdev, + stamps_num, + stamps_rms, + sources, + dtime_diff, + hostname, + good_frac, + code + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!setdiffRunState(config, diff_id, "full")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool diffskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffSkyfile.diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffSkyfile.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "diffSkyfile.tess_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("difftool_skyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool revertdiffskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffSkyfile.diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "diffRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + int num; // Number affected + + // Update state to 'new' + { + psString query = pxDataGet("difftool_revertdiffskyfile_update.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + num = psDBAffectedRows(config->dbh); + + if (num < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } + + // Delete product + { + psString query = pxDataGet("difftool_revertdiffskyfile_delete.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) != num) { + psError(PS_ERR_UNKNOWN, false, "Updated and deleted different number of entries!"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } + + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid diffRun state: %s", state); + return false; + } + + char *query = "UPDATE diffRun SET state = '%s' WHERE diff_id = %"PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, diff_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for diff_id %"PRId64, diff_id); + return false; + } + + return true; +} + +// Generate a single populated run +static bool populatedrun(psArray *list, // List of runs, to print + const char *workdir, // Working directory + const char *skycell_id, // Skycell identifier + const char *tess_id, // Tessellation identifier + const char *label, // label + const char *reduction, // reduction + psS64 input_warp_id, // Warp identifier for input image, PS_MAX_S64 for none + psS64 input_stack_id, // Stack identifier for input image, PS_MAX_S64 for none + psS64 template_warp_id, // Warp identifier for template image, PS_MAX_S64 for none + psS64 template_stack_id, // Stack identifier for template image, PS_MAX_S64 for none + pxConfig *config // Configuration + ) +{ + PS_ASSERT_STRING_NON_EMPTY(workdir, false); + PS_ASSERT_STRING_NON_EMPTY(skycell_id, false); + PS_ASSERT_STRING_NON_EMPTY(tess_id, false); + if ((input_warp_id == PS_MAX_S64 && input_stack_id == PS_MAX_S64) || + (input_warp_id != PS_MAX_S64 && input_stack_id != PS_MAX_S64)) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "One, and only one, input must be defined."); + return false; + } + if ((template_warp_id == PS_MAX_S64 && template_stack_id == PS_MAX_S64) || + (template_warp_id != PS_MAX_S64 && template_stack_id != PS_MAX_S64)) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "One, and only one, template must be defined."); + return false; + } + + // default + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + + diffRunRow *run = diffRunRowAlloc( + 0, // ID + "reg", // state + label, + reduction, + workdir, + NULL, // dvodb + registered, + skycell_id, + tess_id + ); + + if (!run) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc diffRun object"); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!diffRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return true; + } + + // get the assigned diff_id + run->diff_id = psDBLastInsertID(config->dbh); + + // Template + if (!diffInputSkyfileInsert(config->dbh, + run->diff_id, + true, + template_stack_id, + template_warp_id, + skycell_id, + tess_id, + NULL // kind + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // Input + if (!diffInputSkyfileInsert(config->dbh, + run->diff_id, + false, + input_stack_id, + input_warp_id, + skycell_id, + tess_id, + NULL // kind + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + char *query = "UPDATE diffRun SET state = 'new' WHERE diff_id = '%" PRId64 "'"; + if (!p_psDBRunQuery(config->dbh, query, run->diff_id)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to set state to run for diff_id %" PRId64, run->diff_id); + return false; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (list) { + psArrayAdd(list, list->n, run); + } + + psFree(run); + + return true; +} + + +static bool definepoprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); // required options + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); // required options + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_S64(template_warp_id, config->args, "-template_warp_id", false, false); + PXOPT_LOOKUP_S64(template_stack_id, config->args, "-template_stack_id", false, false); + PXOPT_LOOKUP_S64(input_warp_id, config->args, "-input_warp_id", false, false); + PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (template_stack_id && template_warp_id) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Only one template can be defined."); + return false; + } + if (!template_stack_id && !template_warp_id) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "No template has been defined (-template_stack_id or -template_warp_id)"); + return false; + } + + if (input_stack_id && input_warp_id) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Only one input can be defined."); + return false; + } + if (!input_stack_id && !input_warp_id) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "No input has been defined (-input_stack_id or -input_warp_id)"); + return false; + } + + psArray *list = psArrayAllocEmpty(16); // List of runs, to print + + if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction, + input_warp_id ? input_warp_id : PS_MAX_S64, + input_stack_id ? input_stack_id : PS_MAX_S64, + template_warp_id ? template_warp_id : PS_MAX_S64, + template_stack_id ? template_stack_id : PS_MAX_S64, + config)) { + psError(PS_ERR_UNKNOWN, false, "failed to create populated diffRun"); + psFree(list); + return false; + } + + if (!diffRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + psFree(list); + + return true; +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpsToDiff.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpsToDiff.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "warpsToDiff.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "warpsToDiff.filter", "=="); + PXOPT_COPY_STR(config->args, where, "-stack_label", "stacksForDiff.stack_label", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_label", "warpsToDiff.warp_label", "=="); + PXOPT_COPY_STR(config->args, where, "-kind", "warpsToDiff.kind", "=="); + PXOPT_COPY_F32(config->args, where, "-good_frac", "warpsToDiff.good_frac", ">="); + + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); // option + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); // option + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false); + + // find all things to queue + psString query = pxDataGet("difftool_definebyquery.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (newTemplates) { + // Warps that haven't been diffed OR warps that can take a newer template in the diff + psStringAppend(&query, " WHERE (diff_id IS NULL OR best_stack_id > current_stack_id)"); + } else { + // Only warps that haven't been diffed + psStringAppend(&query, " WHERE diff_id IS NULL"); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + psArray *list = psArrayAllocEmpty(16); // List of runs, to print + long numGood = 0; // Number of good rows added + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Output row from query + + // Selected parameters + bool mdok; // Status of MD lookup + const char *skycell_id = psMetadataLookupStr(&mdok, row, "skycell_id"); + if (!mdok || !skycell_id) { + psWarning("skycell_id not found --- ignoring row %ld", i); + continue; + } + const char *tess_id = psMetadataLookupStr(&mdok, row, "tess_id"); + if (!mdok || !tess_id) { + psWarning("tess_id not found --- ignoring row %ld", i); + continue; + } + psS64 stack_id = psMetadataLookupS64(&mdok, row, "best_stack_id"); + if (!mdok) { + psWarning("stack_id not found --- ignoring row %ld", i); + continue; + } + psS64 warp_id = psMetadataLookupS64(&mdok, row, "warp_id"); + if (!mdok) { + psWarning("warp_id not found --- ignoring row %ld", i); + continue; + } + + if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction, warp_id, PS_MAX_S64, PS_MAX_S64, + stack_id, config)) { + psWarning("Unable to add run for %s,%s,%" PRId64 ",%" PRId64, skycell_id, tess_id, + warp_id, stack_id); + psErrorClear(); + continue; + } + + numGood++; + } + psFree(output); + + if (!diffRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + psFree(list); + + return true; +} + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("difftool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffPendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool pendingcleanupskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (diff_id) { + PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + } + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("difftool_pendingcleanupskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffPendingCleanupSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("difftool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/difftool.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/difftool.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/difftool.h (revision 22161) @@ -0,0 +1,44 @@ +/* + * difftool.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef DIFFTOOL_H +#define DIFFTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + DIFFTOOL_MODE_NONE = 0x0, + DIFFTOOL_MODE_DEFINERUN, + DIFFTOOL_MODE_UPDATERUN, + DIFFTOOL_MODE_ADDINPUTSKYFILE, + DIFFTOOL_MODE_INPUTSKYFILE, + DIFFTOOL_MODE_TODIFFSKYFILE, + DIFFTOOL_MODE_ADDDIFFSKYFILE, + DIFFTOOL_MODE_DIFFSKYFILE, + DIFFTOOL_MODE_REVERTDIFFSKYFILE, + DIFFTOOL_MODE_DEFINEPOPRUN, + DIFFTOOL_MODE_DEFINEBYQUERY, + DIFFTOOL_MODE_PENDINGCLEANUPRUN, + DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, + DIFFTOOL_MODE_DONECLEANUP, +} difftoolMode; + +pxConfig *difftoolConfig(pxConfig *config, int argc, char **argv); + +#endif // DIFFTOOL_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/difftoolConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/difftoolConfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/difftoolConfig.c (revision 22161) @@ -0,0 +1,213 @@ +/* + * difftoolConfig.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "difftool.h" + +pxConfig *difftoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -definerun + psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff ID (required)", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + + // -addinputskyfile + psMetadata *addinputskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff ID (required)", 0); + psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "define stack ID", 0); + psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "define warp ID", 0); + psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-kind", 0, "define kind", NULL); + psMetadataAddBool(addinputskyfileArgs, PS_LIST_TAIL, "-template", 0, "this sky cell file is the subtrahend", false); + + // -inputskyfile + psMetadata *inputskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(inputskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddS64(inputskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL); + psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-kind", 0, "search by kind", NULL); + psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-template", 0, "find only subtrahend", false); + psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-input", 0, "find only minuend", false); + psMetadataAddU64(inputskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -todiffskyfile + psMetadata *todiffskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(todiffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddStr(todiffskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddU64(todiffskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(todiffskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -adddiffskyfile + psMetadata *adddiffskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(adddiffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define warp ID (required)", 0); + psMetadataAddS16(adddiffskyfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-uri", 0, "define URI of file", 0); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); + psMetadataAddF64(adddiffskyfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(adddiffskyfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background mean stdev", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-dtime_diff", 0, "define elapsed processing time", NAN); + psMetadataAddS32(adddiffskyfileArgs, PS_LIST_TAIL, "-stamps_num", 0, "define subtraction stamp number", 0); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-stamps_rms", 0, "define subtraction stamp rms", NAN); + psMetadataAddS32(adddiffskyfileArgs, PS_LIST_TAIL, "-sources", 0, "define number of sources", 0); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-good_frac", 0, "define %% of good pixels", NAN); + + // -diffskyfile + psMetadata *diffskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by warp ID", 0); + psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID", NULL); + psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID", NULL); + psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0); + psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-exp_name", 0, "define exposure name", NULL); + psMetadataAddU64(diffskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertdiffskyfile + psMetadata *revertdiffskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(revertdiffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddStr(revertdiffskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddS16(revertdiffskyfileArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -definepoprun + psMetadata *definepoprunArgs = psMetadataAlloc(); + psMetadataAddStr(definepoprunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddTime(definepoprunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definepoprunArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID (required)", NULL); + psMetadataAddStr(definepoprunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID (required)", NULL); + psMetadataAddStr(definepoprunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definepoprunArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); + psMetadataAddBool(definepoprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-template_warp_id", 0, "define warp ID for template", 0); + psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-template_stack_id", 0, "define stack ID for template", 0); + psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-input_warp_id", 0, "define warp ID for input", 0); + psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-input_stack_id", 0, "define stack ID for input", 0); + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-kind", 0, "search by kind", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", 0, "search by label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -pendingcleanuprun + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupskyfile + psMetadata *pendingcleanupskyfileArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definerun", "", DIFFTOOL_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-updaterun", "", DIFFTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-addinputskyfile", "", DIFFTOOL_MODE_ADDINPUTSKYFILE, addinputskyfileArgs); + PXOPT_ADD_MODE("-inputskyfile", "", DIFFTOOL_MODE_INPUTSKYFILE, inputskyfileArgs); + PXOPT_ADD_MODE("-todiffskyfile", "", DIFFTOOL_MODE_TODIFFSKYFILE, todiffskyfileArgs); + PXOPT_ADD_MODE("-adddiffskyfile", "", DIFFTOOL_MODE_ADDDIFFSKYFILE, adddiffskyfileArgs); + PXOPT_ADD_MODE("-diffskyfile", "", DIFFTOOL_MODE_DIFFSKYFILE, diffskyfileArgs); + PXOPT_ADD_MODE("-revertdiffskyfile","", DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileArgs); + PXOPT_ADD_MODE("-definepoprun", "", DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunArgs); + PXOPT_ADD_MODE("-definebyquery", "", DIFFTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); + PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs); + PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", DIFFTOOL_MODE_DONECLEANUP, donecleanupArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/fakemagic =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/fakemagic (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/fakemagic (revision 22161) @@ -0,0 +1,25 @@ +#!/usr/bin/env perl + +# Copyright (C) 2007 Joshua Hoblitt +# +# $Id: fakemagic,v 1.1 2007-11-05 23:29:54 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Getopt::Long qw( GetOptions :config auto_help auto_version pass_through ); +use Pod::Usage qw( pod2usage ); + +my ($output); +GetOptions( + 'output|o=s' => \$output, +) || pod2usage( 2 ); + +pod2usage( -msg => "Required options: --output ", -exitval => 2 ) + unless defined $output; + +open(my $fh, ">$output") or die "can't open file $output: $!"; +close($fh) or die "can't close file $output: $!"; Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/faketool.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/faketool.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/faketool.c (revision 22161) @@ -0,0 +1,1130 @@ +/* + * faketool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "pxdata.h" + +#include "faketool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool pendingexpMode(pxConfig *config); +static bool pendingimfileMode(pxConfig *config); +static bool addprocessedimfileMode(pxConfig *config); +static bool processedimfileMode(pxConfig *config); +static bool revertprocessedimfileMode(pxConfig *config); +static bool updateprocessedimfileMode(pxConfig *config); +static bool blockMode(pxConfig *config); +static bool maskedMode(pxConfig *config); +static bool unmaskedMode(pxConfig *config); +static bool unblockMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupimfileMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); + +static bool fakeProcessedCompleteExp(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) { + psLibInit(NULL); + + pxConfig *config = faketoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(FAKETOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(FAKETOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(FAKETOOL_MODE_PENDINGEXP, pendingexpMode); + MODECASE(FAKETOOL_MODE_PENDINGIMFILE, pendingimfileMode); + MODECASE(FAKETOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileMode); + MODECASE(FAKETOOL_MODE_PROCESSEDIMFILE, processedimfileMode); + MODECASE(FAKETOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileMode); + MODECASE(FAKETOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileMode); + MODECASE(FAKETOOL_MODE_BLOCK, blockMode); + MODECASE(FAKETOOL_MODE_MASKED, maskedMode); + MODECASE(FAKETOOL_MODE_UNMASKED, unmaskedMode); + MODECASE(FAKETOOL_MODE_UNBLOCK, unblockMode); + MODECASE(FAKETOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); + MODECASE(FAKETOOL_MODE_PENDINGCLEANUPIMFILE, pendingcleanupimfileMode); + MODECASE(FAKETOOL_MODE_DONECLEANUP, donecleanupMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + PXOPT_COPY_STR(config->args, where, "-comment", "comment", "LIKE"); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + + // default + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("faketool_find_camrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (pretend) { + // then stop before running the query + fprintf(stderr, "%s\n", query); + psFree(query); + return true; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("faketool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // loop over our list of cam_ids + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + bool status; + psS64 cam_id = psMetadataLookupS64(&status, md, "cam_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for cam_id"); + psFree(output); + return false; + } + + // queue the exp + if (!pxfakeQueueByCamID(config, cam_id, workdir, label, reduction, expgroup, dvodb, tess_id, end_stage)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue cam_id: %" PRId64, cam_id); + psFree(output); + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + PXOPT_COPY_STR(config->args, where, "-label", "fakeRun.label", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + where = NULL; + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + if ((!state) && (!label)) { + psError(PXTOOLS_ERR_DATA, false, "parameters are required"); + psFree(where); + return false; + } + + if (state) { + // set fakeRun.state to state + if (!pxfakeRunSetStateByQuery(config, where, state)) { + psFree(where); + return false; + } + } + + if (label) { + // set fakeRun.label to label + if (!pxfakeRunSetLabelByQuery(config, where, label)) { + psFree(where); + return false; + } + } + + psFree(where); + + return true; +} + + +static bool pendingexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + + psString query = pxDataGet("faketool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fakePendingExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool pendingimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "rawImfile.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + + psString query = pxDataGet("faketool_pendingimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("faketool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fakePendingImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // fake_id, ext_tag, class_id are required + PXOPT_LOOKUP_S64(fake_id, config->args, "-fake_id", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + // optional + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + + PXOPT_LOOKUP_F32(dtime_fake, config->args, "-dtime_fake", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!fakeProcessedImfileInsert(config->dbh, + fake_id, + exp_id, + class_id, + uri, + dtime_fake, + hostname, + path_base, + "full", + code, + NULL // epoch + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // XXX I've decided to make the transaction cover the Exp migration as + // well. Otherwise, if the last imfile in an exp is moved and the exp + // migration fails then the data base is left in a satuation where the exp + // migration can't happen. + + if (!fakeProcessedCompleteExp(config)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "rawImfile.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + + psString query = pxDataGet("faketool_processedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "fakeProcessedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND fakeProcessedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND fakeProcessedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("faketool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fakeProcessedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool revertprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "fakeRun.label", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("faketool_revertprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + +static bool updateprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(code, config->args, "-code", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + if (!pxSetFaultCode(config->dbh, "fakeProcessedImfile", where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree(where); + return false; + } + psFree(where); + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + if (!fakeMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (where->list->n < 1) { + psFree(where); + where = NULL; + } + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM fakeMask"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psFree(where); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("faketool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fakeMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +// return the list of labels which are NOT blocked +static bool unmaskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (where->list->n < 1) { + psFree(where); + where = NULL; + } + + psString query = pxDataGet("faketool_unmasked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, "fakeUnmask"); + psFree(where); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("faketool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fakeUnmask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + char *query = "DELETE FROM fakeMask WHERE label = '%s'"; + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("faketool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("faketool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fakePendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool pendingcleanupimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(fake_id, config->args, "-fake_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (fake_id) { + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + } + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("faketool_pendingcleanupimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("faketool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fakePendingCleanupImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("faketool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("faketool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fakeDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool fakeProcessedCompleteExp(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // look for completed fakePendingExp + // migrate them to fakeProccessedExp & camPendingExp + psString query = pxDataGet("faketool_completely_processed_exp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("faketool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + fakeRunRow *fakeRun = fakeRunObjectFromMetadata(row); + // set fakeRun.state to 'full' + if (!pxfakeRunSetState(config, fakeRun->fake_id, "full")) { + psError(PS_ERR_UNKNOWN, false, "failed to change fakeRun.state for fake_id: %" PRId64, fakeRun->fake_id); + psFree(fakeRun); + psFree(output); + return false; + } + + // should we stop here or proceed on to the cam stage? + // NULL for end_stage means go as far as possible + if ((fakeRun->end_stage && psStrcasestr(fakeRun->end_stage, "fake")) || !fakeRun->tess_id) { + psFree(fakeRun); + continue; + } + // else continue on... + + // camQueueFakeID() can only be run after fakeRun.state has been set to + // stop + if (!pxwarpQueueByFakeID(config, + fakeRun->fake_id, + fakeRun->workdir, + fakeRun->label, + fakeRun->dvodb, + fakeRun->tess_id, + fakeRun->end_stage + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to queue camPendingExp"); + psFree(fakeRun); + psFree(output); + return false; + } + psFree(fakeRun); + } + + psFree(output); + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/faketool.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/faketool.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/faketool.h (revision 22161) @@ -0,0 +1,47 @@ +/* + * faketool.h + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef FAKETOOL_H +#define FAKETOOL_H 1 + +#include "pxtools.h" + +typedef enum { + FAKETOOL_MODE_NONE = 0x0, + FAKETOOL_MODE_DEFINEBYQUERY, + FAKETOOL_MODE_UPDATERUN, + FAKETOOL_MODE_PENDINGIMFILE, + FAKETOOL_MODE_PENDINGEXP, + FAKETOOL_MODE_ADDPROCESSEDIMFILE, + FAKETOOL_MODE_PROCESSEDIMFILE, + FAKETOOL_MODE_REVERTPROCESSEDIMFILE, + FAKETOOL_MODE_UPDATEPROCESSEDIMFILE, + FAKETOOL_MODE_BLOCK, + FAKETOOL_MODE_MASKED, + FAKETOOL_MODE_UNMASKED, + FAKETOOL_MODE_UNBLOCK, + FAKETOOL_MODE_RETRYPROCESSEDIMFILE, + FAKETOOL_MODE_PENDINGCLEANUPRUN, + FAKETOOL_MODE_PENDINGCLEANUPIMFILE, + FAKETOOL_MODE_DONECLEANUP, +} FAKETOOLMode; + +pxConfig *faketoolConfig(pxConfig *config, int argc, char **argv); + +#endif // FAKETOOL_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/faketoolConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/faketoolConfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/faketoolConfig.c (revision 22161) @@ -0,0 +1,320 @@ +/* + * faketoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "faketool.h" + +pxConfig *faketoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + psMetadata *queueArgs = psMetadataAlloc(); + // XXX need to allow multiple exp_ids + psMetadataAddS64(queueArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(queueArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(queueArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", "object"); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-ra_min", 0, "define min", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-ra_max", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-decl_min", 0, "define min", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-decl_max", 0, "define max", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define min fraction of saturated pixels", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_min", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_max", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-alt_min", 0, "define min", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-alt_max", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-az_min", 0, "define min", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-az_max", 0, "define max", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_expgroup", 0, "define exposure group", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessellation identifier", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddBool(queueArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(queueArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-fake_id", 0, "search by fake ID", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", "object"); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ra_min", 0, "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ra_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-decl_min", 0, "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-decl_max", 0, "define max", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define min fraction of saturated pixels", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_min", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-alt_min", 0, "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-alt_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-az_min", 0, "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-az_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-fake_id", 0, "search by fake ID", 0); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chiptool ID", 0); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -pendingimfile + psMetadata *pendingimfileArgs = psMetadataAlloc(); + psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "search by fake ID", 0); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-inst", 0, "search by camera of interest", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-filter", 0, "search by filter of interest", NULL); + psMetadataAddU64(pendingimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addprocessedimfile + psMetadata *addprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "define fake ID (required)", 0); + psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID (required)", 0); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID (required)", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-uri", 0, "define URL", NULL); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-dtime_fake", 0, "define elapsed time for detrend (seconds)", NAN); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -processedimfile + psMetadata *processedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "define fake ID", 0); + psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-inst", 0, "define camera of interest", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-filter", 0, "define filter of interest", NULL); + psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updateprocessedimfile + psMetadata *updateprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "search by fake ID", 0); + psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code (required)", 0); + + // -revertprocessedimfile + psMetadata *revertprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "search by fake ID", 0); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(revertprocessedimfileArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(revertprocessedimfileArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", "object"); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-ra_min", 0, "define min", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-ra_max", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-decl_min", 0, "define min", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-decl_max", 0, "define max", NAN); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min", NAN); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max", NAN); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define min fraction of saturated pixels", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-bg_min", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-bg_max", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-alt_min", 0, "define min", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-alt_max", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-az_min", 0, "define min", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-az_max", 0, "define max", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-ccd_temp_min",0, "define min ccd tempature", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-ccd_temp_max",0, "define max ccd tempature", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + + psMetadataAddBool(revertprocessedimfileArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddS16(revertprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddStr(maskedArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(maskedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -unmasked + psMetadata *unmaskedArgs = psMetadataAlloc(); + psMetadataAddStr(unmaskedArgs, PS_LIST_TAIL, "-label", 0, "restrict to specified label", NULL); + psMetadataAddBool(unmaskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(unmaskedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + + // -pendingcleanuprun + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupimfile + psMetadata *pendingcleanupimfileArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "search by chip ID", 0); + psMetadataAddBool(pendingcleanupimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from raw exposures", FAKETOOL_MODE_DEFINEBYQUERY, queueArgs); + PXOPT_ADD_MODE("-updaterun", "change fake run properties", FAKETOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingexp", "show pending exposures", FAKETOOL_MODE_PENDINGEXP, pendingexpArgs); + PXOPT_ADD_MODE("-pendingimfile", "show pending imfiles", FAKETOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + PXOPT_ADD_MODE("-addprocessedimfile", "add a processed imfile", FAKETOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); + PXOPT_ADD_MODE("-processedimfile", "show processed imfiles", FAKETOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); + PXOPT_ADD_MODE("-updateprocessedimfile","change procesed imfile properties", FAKETOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileArgs); + PXOPT_ADD_MODE("-revertprocessedimfile", "undo a processed imfile", FAKETOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileArgs); + PXOPT_ADD_MODE("-block", "set a label block", FAKETOOL_MODE_BLOCK, blockArgs); + PXOPT_ADD_MODE("-masked", "show blocked labels", FAKETOOL_MODE_MASKED, maskedArgs); + PXOPT_ADD_MODE("-unmasked", "", FAKETOOL_MODE_UNMASKED, unmaskedArgs); + PXOPT_ADD_MODE("-unblock", "remove a label block", FAKETOOL_MODE_UNBLOCK, unblockArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", FAKETOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); + PXOPT_ADD_MODE("-pendingcleanupimfile", "show runs that need to be cleaned up", FAKETOOL_MODE_PENDINGCLEANUPIMFILE, pendingcleanupimfileArgs); + PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", FAKETOOL_MODE_DONECLEANUP, donecleanupArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/flatcorr.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/flatcorr.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/flatcorr.c (revision 22161) @@ -0,0 +1,663 @@ +/* + * flatcorr.c + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * Copyright (C) 2008 Eugene Magnier + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include + +#include "pxtools.h" +#include "pxchip.h" +#include "flatcorr.h" + +static bool definebyqueryMode(pxConfig *config); +static bool definerunMode(pxConfig *config); +static bool addchipMode(pxConfig *config); +static bool addcameraMode(pxConfig *config); +static bool pendingprocessMode(pxConfig *config); +static bool addprocessMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool inputexpMode(pxConfig *config); +static bool inputimfileMode(pxConfig *config); + +static bool setflatcorrRunState(pxConfig *config, psS64 corr_id, const char *state); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = flatcorrConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(FLATCORR_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(FLATCORR_MODE_DEFINERUN, definerunMode); + MODECASE(FLATCORR_MODE_ADDCHIP, addchipMode); + MODECASE(FLATCORR_MODE_ADDCAMERA, addcameraMode); + MODECASE(FLATCORR_MODE_PENDINGPROCESS, pendingprocessMode); + MODECASE(FLATCORR_MODE_ADDPROCESS, addprocessMode); + MODECASE(FLATCORR_MODE_UPDATERUN, updaterunMode); + MODECASE(FLATCORR_MODE_INPUTEXP, inputexpMode); + MODECASE(FLATCORR_MODE_INPUTIMFILE, inputimfileMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-set_filter", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(region, config->args, "-set_region", false, false); + // XXX probably should make the region in -set_region match ra_min, ra_max, etc + + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("chiptool_find_rawexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("flatcorr", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + for (long i = 0; i < psArrayLength(output); i++) { + // negative simple so the default is true + if (!ippdbPrintMetadataRaw(stdout, output->data[i], !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + psFree(output); + return true; + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // create a new flatcorrRun + if (!flatcorrRunInsert( + config->dbh, + 0, // corr_id + dvodb, + filter, + "reg", // state + workdir, + label, + reduction, + region, + NULL, + 0 + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // figure out the ID of the flatcorrRun we just created + psS64 corr_id = psDBLastInsertID(config->dbh); + + // loop over our list of exp_ids + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + bool status; + psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id"); + if (!status) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_id"); + psFree(output); + return false; + } + + // queue the exp : force this to stop at the chip stage + psS64 chip_id = pxchipQueueByExpTag(config, exp_id, workdir, label, reduction, expgroup, dvodb, tess_id, "chip"); + if (!chip_id) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue exp_id: %" PRId64, exp_id); + psFree(output); + return false; + } + + // add a flatcorrChipLink to the flatcorr Run we just created + if (!flatcorrChipLinkInsert(config->dbh, corr_id, chip_id)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + } + psFree(output); + + // set the flatcorrRun to a state of 'new' + if (!setflatcorrRunState(config, corr_id, "new")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set run state"); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool definerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-set_filter", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(region, config->args, "-set_region", false, false); + // XXX probably should make the region in -set_region match ra_min, ra_max, etc + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // create a new flatcorrRun + flatcorrRunRow *row = flatcorrRunRowAlloc( + 0, // corr_id + dvodb, + filter, + "reg", // state + workdir, + label, + reduction, + region, + NULL, // hostname + 0 // fault + ); + + // create a new flatcorrRun + if (!flatcorrRunInsertObject(config->dbh,row)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // figure out the ID of the flatcorrRun we just created + psS64 corr_id = psDBLastInsertID(config->dbh); + row->corr_id = corr_id; + + flatcorrRunPrintObject (stdout, row, !simple); + return true; +} + +static bool addchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + + if (!flatcorrChipLinkInsert(config->dbh, corr_id, chip_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +// select the flatcorr chip runs that have completed and for which there is no camera entry +// queue a new camera run for them +static bool addcameraMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + psString query = pxDataGet("flatcorr_chiprundone.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("flatcorr", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "flatcorr_addcamera", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // loop over our list of chipRun rows + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + bool status; + psS64 corr_id = psMetadataLookupS64(&status, md, "corr_id"); + if (!status) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for corr_id"); + psFree(output); + return false; + } + + chipRunRow *row = chipRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxcamQueueByChipID( + config, + row->chip_id, + row->workdir, + row->label, + row->reduction, + row->expgroup, + row->dvodb, + row->tess_id, + "camera")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue chip_id: %" PRId64, row->chip_id); + psFree(row); + psFree(output); + return false; + } + + // figure out the ID of the flatcorrRun we just created + psS64 cam_id = psDBLastInsertID(config->dbh); + + // add the camRun entry to the flatcorrCamLink table: + if (!flatcorrCamLinkInsert(config->dbh, corr_id, row->chip_id, cam_id)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return false; +} + +// select the flatcorr chip runs that have completed and for which there is no camera entry +// queue a new camera run for them +static bool pendingprocessMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); + + psString query = pxDataGet("flatcorr_pendingprocess.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("flatcorr", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "flatcorrPending", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + return true; +} + +// select the flatcorr chip runs that have completed and for which there is no camera entry +// queue a new camera run for them +static bool inputexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); + + char *query = psStringCopy ("SELECT * FROM flatcorrChipLink WHERE corr_id = %" PRId64); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query, corr_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("flatcorr", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "flatcorrPending", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + return true; +} + +// select the flatcorr chip runs that have completed and for which there is no camera entry +// queue a new camera run for them +static bool inputimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipProcessedImfile.chip_id", "=="); + + psString query = pxDataGet("flatcorr_inputimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query, chip_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("flatcorr", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "flatcorrPending", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + return true; +} + +// XXX need a fault state +static bool addprocessMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + char *query = "UPDATE flatcorrRun SET state = 'full', hostname = '%s', fault = '%hd' WHERE corr_id = %" PRId64; + + if (!p_psDBRunQuery(config->dbh, query, hostname, code, corr_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to change state for corr_id %" PRId64, corr_id); + return false; + } + + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + + if (!setflatcorrRunState(config, corr_id, state)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set run state"); + return false; + } + + return true; +} + +static bool setflatcorrRunState(pxConfig *config, psS64 corr_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!strcmp(state, "reg") && + !strcmp(state, "new") && + !strcmp(state, "full")) + { + psError(PS_ERR_UNKNOWN, false, "invalid state: %s", state); + return false; + } + + char *query = "UPDATE flatcorrRun SET state = '%s' WHERE corr_id = %" PRId64; + + if (!p_psDBRunQuery(config->dbh, query, state, corr_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to change state for corr_id %" PRId64, corr_id); + return false; + } + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/flatcorr.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/flatcorr.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/flatcorr.h (revision 22161) @@ -0,0 +1,40 @@ +/* + * flatcorr.h + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef FLATCORR_H +#define FLATCORR_H 1 + +#include "pxtools.h" + +typedef enum { + FLATCORR_MODE_NONE = PXTOOL_MODE_NONE, + FLATCORR_MODE_DEFINEBYQUERY, + FLATCORR_MODE_DEFINERUN, + FLATCORR_MODE_ADDCHIP, + FLATCORR_MODE_ADDCAMERA, + FLATCORR_MODE_PENDINGPROCESS, + FLATCORR_MODE_ADDPROCESS, + FLATCORR_MODE_UPDATERUN, + FLATCORR_MODE_INPUTEXP, + FLATCORR_MODE_INPUTIMFILE +} flatcorrMode; + +pxConfig *flatcorrConfig(pxConfig *config, int argc, char **argv); + +#endif // FLATCORR_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/flatcorrConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/flatcorrConfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/flatcorrConfig.c (revision 22161) @@ -0,0 +1,152 @@ +/* + * flatcorrConfig.c + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "flatcorr.h" + +pxConfig *flatcorrConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + // XXX need to allow multiple exp_ids + pxchipSetSearchArgs (definebyqueryArgs); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction",0, "define reduction class", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_expgroup", 0, "define exposure group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_filter", 0, "define filter", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessalation", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_region", 0, "define region", NULL); + + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "print the exposures that would be included in the detrend run and exit", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -definerun + psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_expgroup", 0, "define exposure group", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_filter", 0, "define filter", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessalation", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_region", 0, "define region", NULL); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addchip + psMetadata *addchipArgs = psMetadataAlloc(); + psMetadataAddS64(addchipArgs, PS_LIST_TAIL, "-corr_id", 0, "define Flat Correction ID (required)", 0); + psMetadataAddS64(addchipArgs, PS_LIST_TAIL, "-chip_id", 0, "define Chip ID (required)", 0); + + // -addcamera + psMetadata *addcameraArgs = psMetadataAlloc(); + psMetadataAddU64 (addcameraArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(addcameraArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(addcameraArgs, PS_LIST_TAIL, "-pretend", 0, "use the simple output format", false); + + // -pendingprocess + psMetadata *pendingprocessArgs = psMetadataAlloc(); + psMetadataAddU64 (pendingprocessArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingprocessArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addprocess (XXX need to add fault and stats) + psMetadata *addprocessArgs = psMetadataAlloc(); + psMetadataAddS64 (addprocessArgs, PS_LIST_TAIL, "-corr_id", 0, "add complete run for specified corr_id (required)", 0); + psMetadataAddStr (addprocessArgs, PS_LIST_TAIL, "-hostname", 0, "set hostname", NULL); + psMetadataAddS16 (addprocessArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-corr_id", 0, "define correction id (required)", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + + // -inputexp + psMetadata *inputexpArgs = psMetadataAlloc(); + psMetadataAddS64(inputexpArgs, PS_LIST_TAIL, "-corr_id", 0, "define correction id (required)", 0); + psMetadataAddBool(inputexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(inputexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -inputimfile + psMetadata *inputimfileArgs = psMetadataAlloc(); + psMetadataAddS64(inputimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "define chip id (required)", 0); + psMetadataAddBool(inputimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(inputimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create a new, populated flat correction run", FLATCORR_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definerun", "create a new, empty flat correction run", FLATCORR_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-addchip", "add a chip to a flat correction run", FLATCORR_MODE_ADDCHIP, addchipArgs); + PXOPT_ADD_MODE("-addcamera", "migrate completed chips to camera stage analysis", FLATCORR_MODE_ADDCAMERA, addcameraArgs); + PXOPT_ADD_MODE("-pendingprocess", "show flat correction runs needing to be processed", FLATCORR_MODE_PENDINGPROCESS, pendingprocessArgs); + PXOPT_ADD_MODE("-addprocess", "report completed flat correction analysis", FLATCORR_MODE_ADDPROCESS, addprocessArgs); + PXOPT_ADD_MODE("-updaterun", "change a flat calibration run's state", FLATCORR_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-inputexp", "list exposures for a correction run", FLATCORR_MODE_INPUTEXP, inputexpArgs); + PXOPT_ADD_MODE("-inputimfile", "list imfiles for a chip run", FLATCORR_MODE_INPUTIMFILE, inputimfileArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/magictool.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/magictool.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/magictool.c (revision 22161) @@ -0,0 +1,1278 @@ +/* + * magictool.c + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "magictool.h" + +static bool definebyqueryMode(pxConfig *config); +static psS64 definerunMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool addinputskyfileMode(pxConfig *config); +static bool inputskyfileMode(pxConfig *config); +static bool totreeMode(pxConfig *config); +static bool inputtreeMode(pxConfig *config); +static bool reverttreeMode(pxConfig *config); +static bool toprocessMode(pxConfig *config); +static bool addresultMode(pxConfig *config); +static bool revertnodeMode(pxConfig *config); +static bool inputsMode(pxConfig *config); +static bool tomaskMode(pxConfig *config); +static bool addmaskMode(pxConfig *config); +static bool revertmaskMode(pxConfig *config); +static bool maskMode(pxConfig *config); + +static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state); +static bool parseAndInsertNodeDeps(pxConfig *config, psS64 magic_id, const char *filename); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = magictoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(MAGICTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(MAGICTOOL_MODE_DEFINERUN, definerunMode); + MODECASE(MAGICTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(MAGICTOOL_MODE_ADDINPUTSKYFILE,addinputskyfileMode); + MODECASE(MAGICTOOL_MODE_INPUTSKYFILE, inputskyfileMode); + MODECASE(MAGICTOOL_MODE_TOTREE, totreeMode); + MODECASE(MAGICTOOL_MODE_INPUTTREE, inputtreeMode); + MODECASE(MAGICTOOL_MODE_REVERTTREE, reverttreeMode); + MODECASE(MAGICTOOL_MODE_TOPROCESS, toprocessMode); + MODECASE(MAGICTOOL_MODE_ADDRESULT, addresultMode); + MODECASE(MAGICTOOL_MODE_REVERTNODE, revertnodeMode); + MODECASE(MAGICTOOL_MODE_INPUTS, inputsMode); + MODECASE(MAGICTOOL_MODE_TOMASK, tomaskMode); + MODECASE(MAGICTOOL_MODE_ADDMASK, addmaskMode); + MODECASE(MAGICTOOL_MODE_REVERTMASK, revertmaskMode); + MODECASE(MAGICTOOL_MODE_MASK, maskMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // Required + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); + + // Optional + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // Create temporary table of the best diffs + { + psString query = pxDataGet("magictool_definebyquery_temp_create.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + } + + // Insert list of best diffs into temporary table + { + psString query = pxDataGet("magictool_definebyquery_temp_insert.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + psString groupby = pxDataGet("magictool_definebyquery_temp_insert_groupby.sql"); + if (!groupby) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(query); + return false; + } + + psStringAppend(&query, " %s", groupby); + psFree(groupby); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + } + + // Get list of exposures ready to magic + { + psString query = pxDataGet("magictool_definebyquery_select_part1.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + { + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + } + + psString part2 = pxDataGet("magictool_definebyquery_select_part2.sql"); + if (!part2) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psStringAppend(&query, " %s", part2); + psFree(part2); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + } + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psString insert = pxDataGet("magictool_definebyquery_insert.sql"); // Insert query + + psArray *list = psArrayAllocEmpty(16); // List of runs, to print + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; // Row of interest + psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier + + // create a new magicRun for this group + magicRunRow *run = magicRunRowAlloc(0, exp_id, "run", workdir, "dirty", label, dvodb, registered, 0); + if (!run) { + psAbort("failed to alloc magicRun object"); + } + + if (!magicRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + psFree(insert); + psFree(output); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psS64 magic_id = psDBLastInsertID(config->dbh); // Assigned identifier + run->magic_id = magic_id; + + psArrayAdd(list, list->n, run); + psFree(run); + + // Create a suitable insertion query for this run + psString thisInsert = psStringCopy(insert); + { + psString idString = NULL; + psStringAppend(&idString, "%" PRId64, magic_id); + psStringSubstitute(&thisInsert, idString, "@MAGIC_ID@"); + psFree(idString); + } + { + psString idString = NULL; + psStringAppend(&idString, "%" PRId64, exp_id); + psStringSubstitute(&thisInsert, idString, "@EXP_ID@"); + psFree(idString); + } + + if (!p_psDBRunQuery(config->dbh, thisInsert, magic_id, exp_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(thisInsert); + psFree(insert); + psFree(output); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(thisInsert); + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(output); + + if (!magicRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + + psFree(list); + + return true; +} + +static psS64 definerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + + // optional + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + magicRunRow *run = magicRunRowAlloc( + 0, // ID + exp_id, + "reg", // state + workdir, + "dirty", // workdir_state + label, + dvodb, + registered, + 0 + ); + + if (!run) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc magicRun object"); + return false; + } + if (!magicRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return false; + } + + // get the assigned warp_id + psS64 magic_id = psDBLastInsertID(config->dbh); + run->magic_id = magic_id; + + if (!magicRunPrintObject(stdout, run, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(run); + return false; + } + + psFree(run); + + return magic_id; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + + if (state) { + // set detRun.state to state + return setmagicRunState(config, magic_id, state); + } + + return true; +} + + +static bool addinputskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + PXOPT_LOOKUP_STR(node, config->args, "-node", true, false); + + magicInputSkyfileInsert( + config->dbh, + magic_id, + diff_id, + node + ); + + return true; +} + + +static bool inputskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-node", "node", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("magictool_inputskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "magicInputSkyfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "magicInputSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool totreeMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // look for "inputs" that need to processed + psString query = pxDataGet("magictool_totree.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "totree", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool inputtreeMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); + + // Optional values + PXOPT_LOOKUP_STR(dep_file, config->args, "-dep_file", false, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (code > 0) { + char *query = "UPDATE magicRun SET fault = %d, state = 'stop' WHERE magic_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, code, magic_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to set fault for magic_id %" PRId64, magic_id); + return false; + } + return true; + } + + if (!parseAndInsertNodeDeps(config, magic_id, dep_file)) { + psError(PS_ERR_UNKNOWN, false, "failed to parse file"); + return false; + } + + return true; +} + +static bool reverttreeMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'run' WHERE fault != 0"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "failed to revert"); + return false; + } + return true; +} + + +static bool inputsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + PXOPT_COPY_STR(config->args, where, "-node", "node", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("magictool_inputs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "magicNode", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +bool findBaseNodes(void *arg, pxNode *node) +{ + bool status = false; + psS64 done = psMetadataLookupS64(&status, node->data, "done"); + if (!status) { + psAbort("failed to lookup value for done column"); + } + + if ((!pxNodeHasChildren(node)) && (!done)) { + // if this node has no child and it's not 'done', then push it's data + // onto the void *array + psArrayAdd((psArray *)arg, 0, node->data); + return false; + } + + return true; +} + + +bool findReadyNodes(void *arg, pxNode *node) +{ + if (!node) { + // It's not there --- must have failed + return false; + } + + if (!node->data) { + // It's a leaf node, not done + return false; + } + + if (psMetadataLookupBool(NULL, node->data, "done")) { + // It's already done + return true; + } + + if (pxNodeHasChildren(node)) { + psListIterator *iter = psListIteratorAlloc(node->children, 0, false); + psMetadata *work = psMetadataCopy(NULL, node->data); + psMetadataRemoveKey(work, "dep"); + psMetadataRemoveKey(work, "done"); + pxNode *child = NULL; + while ((child = psListGetAndIncrement(iter))) { + psMetadata *data = child->data; + if (!data) { + // Child is a leaf node, not done + psFree(iter); + psFree(work); + return false; + } + + bool status = false; + psS32 done = psMetadataLookupS32(&status, data, "done"); + if (!status) { + psAbort("failed to lookup value for done column"); + } + psS16 bad = psMetadataLookupS16(&status, data, "bad"); + if (!status) { + psAbort("failed to lookup value for bad column"); + } + + if (!done || bad) { + // if a child isn't "done", give up on this node and continue + // to crawl the tree + psFree(iter); + psFree(work); + return true; + } + } + psFree(iter); + // if all this nodes children are done, then push it's data onto the + // void *array + psArrayAdd((psArray *)arg, 0, work); + psFree(work); + return false; + } + + return true; +} + + +static bool toprocessMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // look for "inputs" that need to processed + psString query = pxDataGet("magictool_toprocess_inputs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = NULL; + if (psListLength(where->list)) { + whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + // psFree(output); + // return true; + } + + // look for tree nodes that need to be processed + query = pxDataGet("magictool_toprocess_tree.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (whereClause) { + psStringAppend(&query, " %s", whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *magicTree = p_psDBFetchResult(config->dbh); + if (!magicTree) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(magicTree)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(magicTree); + return true; + } + + psHash *forest = psHashAlloc(psArrayLength(magicTree)); + + // convert the array of metadata into a pxTree structure + for (long i = 0; i < psArrayLength(magicTree); i++) { + bool status; + psString node = psMetadataLookupStr(&status, magicTree->data[i], "node"); + if (!status) { + psAbort("failed to lookup value for node column"); + } + + psString dep = psMetadataLookupStr(&status, magicTree->data[i], "dep"); + if (!status) { + psAbort("failed to lookup value for dep column"); + } + + pxTreeBuilder(forest, node, dep, magicTree->data[i]); + + } + psFree(magicTree); + + // find the root of the tree + pxNode *root = psMemIncrRefCounter(psHashLookup(forest, "root")); + psFree(forest); + // pxTreePrint(stdout, root); + + // crawl through the tree and looking for nodes with children that are all + // "done" + pxTreeCrawl(root, findReadyNodes, output); + psFree(root); + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + psFree(whereClause); + + return true; +} + + +static bool addresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); + PXOPT_LOOKUP_STR(node, config->args, "-node", true, false); + + // optional + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (!magicNodeResultInsert(config->dbh, + magic_id, + node, + uri, + code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool revertnodeMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + PXOPT_COPY_STR(config->args, where, "-node", "node", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = psStringCopy("DELETE FROM magicNodeResult WHERE fault != 0"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "failed to revert"); + return false; + } + return true; +} + + +static bool tomaskMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // look for "inputs" that need to processed + psString query = pxDataGet("magictool_tomask.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "tomask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool addmaskMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); + + // optional + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_S32(streaks, config->args, "-streaks", false, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!magicMaskInsert(config->dbh, + magic_id, + uri, + streaks, + code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + // Set the magicRun state + psString query = pxDataGet("magictool_addmask.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + // manually add constraint + psStringAppend(&query, " AND magic_id = %" PRId64, magic_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool revertmaskMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // Set to "run" + { + psString query = psStringCopy("UPDATE magicRun JOIN magicMask USING(magic_id) " + "SET magicRun.state = 'run' WHERE magicMask.fault != 0"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "magicMask"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "failed to revert"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psFree(query); + } + + // Delete failed attempt at mask + { + psString query = psStringCopy("DELETE FROM magicMask WHERE fault != 0"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "magicMask"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "failed to revert"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psFree(query); + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(where); + + return true; +} + + +static bool maskMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("magictool_mask.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "magicMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "reg", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid magicRun state: %s", state); + return false; + } + + char *query = "UPDATE magicRun SET state = '%s' WHERE magic_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, magic_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for magic_id %" PRId64, magic_id); + return false; + } + + return true; +} + +static bool parseAndInsertNodeDeps(pxConfig *config, psS64 magic_id, const char *filename) +{ + unsigned int nFail = 0; + psMetadata *deps = psMetadataConfigRead(NULL, &nFail, filename, false); + if (!deps) { + psError(PS_ERR_UNKNOWN, false, "failed to parse file: %s", filename); + return false; + } + if (nFail) { + psError(PS_ERR_UNKNOWN, false, "there were %d errors parsing file: %s", nFail, filename); + psFree(deps); + return false; + } + + psMetadataItem *item = NULL; + psMetadataIterator *iter = psMetadataIteratorAlloc(deps, 0, NULL); + while ((item = psMetadataGetAndIncrement(iter))) { + if (item->type != PS_DATA_STRING) { + psError(PS_ERR_UNKNOWN, false, "file: %s is in the wrong format", filename); + psFree(iter); + psFree(deps); + return false; + } + + char *name = item->name; + char *dependsOn = item->data.str; + + if (!magicTreeInsert( + config->dbh, + magic_id, + name, + dependsOn + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(iter); + psFree(deps); + return false; + } + } + psFree(iter); + psFree(deps); + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/magictool.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/magictool.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/magictool.h (revision 22161) @@ -0,0 +1,47 @@ +/* + * magictool.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef MAGICTOOL_H +#define MAGICTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + MAGICTOOL_MODE_NONE = 0x0, + MAGICTOOL_MODE_DEFINEBYQUERY, + MAGICTOOL_MODE_DEFINERUN, + MAGICTOOL_MODE_UPDATERUN, + MAGICTOOL_MODE_ADDINPUTSKYFILE, + MAGICTOOL_MODE_INPUTSKYFILE, + MAGICTOOL_MODE_TOTREE, + MAGICTOOL_MODE_INPUTTREE, + MAGICTOOL_MODE_REVERTTREE, + MAGICTOOL_MODE_TOPROCESS, + MAGICTOOL_MODE_INPUTS, + MAGICTOOL_MODE_ADDRESULT, + MAGICTOOL_MODE_REVERTNODE, + MAGICTOOL_MODE_TOMASK, + MAGICTOOL_MODE_ADDMASK, + MAGICTOOL_MODE_REVERTMASK, + MAGICTOOL_MODE_MASK, +} MAGICtoolMode; + +pxConfig *magictoolConfig(pxConfig *config, int argc, char **argv); + +#endif // MAGICTOOL_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/magictoolConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/magictoolConfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/magictoolConfig.c (revision 22161) @@ -0,0 +1,197 @@ +/* + * magictoolConfig.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "magictool.h" + +pxConfig *magictoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "limit good_frac", NAN); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -definerun + psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-exp_id", 0, "define exp_id (required)", 0); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); + psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + + // -addinputskyfile + psMetadata *addinputskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); + psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define difftool ID (required)", 0); + psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-node", 0, "define symbolic node name (required)", NULL); + + // -inputskyfile + psMetadata *inputskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(inputskyfileArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0); + psMetadataAddS64(inputskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by difftool ID", 0); + psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-node", 0, "search by symbolic node name", NULL); + psMetadataAddU64(inputskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -totree + psMetadata *totreeArgs = psMetadataAlloc(); + psMetadataAddS64(totreeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); + psMetadataAddU64(totreeArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(totreeArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -inputtree + psMetadata *inputtreeArgs = psMetadataAlloc(); + psMetadataAddS64(inputtreeArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); + psMetadataAddStr(inputtreeArgs, PS_LIST_TAIL, "-dep_file", 0, "order of operations dep. file", NULL); + psMetadataAddS16(inputtreeArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -reverttree + psMetadata *reverttreeArgs = psMetadataAlloc(); + psMetadataAddS64(reverttreeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0); + psMetadataAddS16(reverttreeArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -inputs + psMetadata *inputsArgs = psMetadataAlloc(); + psMetadataAddS64(inputsArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0); + psMetadataAddStr(inputsArgs, PS_LIST_TAIL, "-node", 0, "search by symbolic node name", NULL); + psMetadataAddU64(inputsArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(inputsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -tooprocess + psMetadata *toprocessArgs = psMetadataAlloc(); + psMetadataAddS64(toprocessArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); + psMetadataAddU64(toprocessArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(toprocessArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addresult + psMetadata *addresultArgs = psMetadataAlloc(); + psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-node", 0, "define symbolic node name (required)", NULL); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL); + psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -revertnode + psMetadata *revertnodeArgs = psMetadataAlloc(); + psMetadataAddS64(revertnodeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0); + psMetadataAddStr(revertnodeArgs, PS_LIST_TAIL, "-node", 0, "search by node name", NULL); + psMetadataAddS16(revertnodeArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -tomask + psMetadata *tomaskArgs = psMetadataAlloc(); + psMetadataAddU64(tomaskArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tomaskArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addmask + psMetadata *addmaskArgs = psMetadataAlloc(); + psMetadataAddS64(addmaskArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); + psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL); + psMetadataAddS32(addmaskArgs, PS_LIST_TAIL, "-streaks", 0, "define number of streaks", 0); + psMetadataAddS16(addmaskArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -revertmask + psMetadata *revertmaskArgs = psMetadataAlloc(); + psMetadataAddS64(revertmaskArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0); + psMetadataAddS16(revertmaskArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -mask + psMetadata *maskArgs = psMetadataAlloc(); + psMetadataAddS64(maskArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID", 0); + psMetadataAddU64(maskArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(maskArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "", MAGICTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definerun", "", MAGICTOOL_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-updaterun", "", MAGICTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-addinputskyfile", "", MAGICTOOL_MODE_ADDINPUTSKYFILE, addinputskyfileArgs); + PXOPT_ADD_MODE("-inputskyfile", "", MAGICTOOL_MODE_INPUTSKYFILE, inputskyfileArgs); + PXOPT_ADD_MODE("-totree", "", MAGICTOOL_MODE_TOTREE, totreeArgs); + PXOPT_ADD_MODE("-inputtree", "", MAGICTOOL_MODE_INPUTTREE, inputtreeArgs); + PXOPT_ADD_MODE("-reverttree", "", MAGICTOOL_MODE_REVERTTREE, reverttreeArgs); + PXOPT_ADD_MODE("-toprocess", "", MAGICTOOL_MODE_TOPROCESS, toprocessArgs); + PXOPT_ADD_MODE("-inputs", "", MAGICTOOL_MODE_INPUTS, inputsArgs); + PXOPT_ADD_MODE("-addresult", "", MAGICTOOL_MODE_ADDRESULT, addresultArgs); + PXOPT_ADD_MODE("-revertnode", "", MAGICTOOL_MODE_REVERTNODE, revertnodeArgs); + PXOPT_ADD_MODE("-tomask", "", MAGICTOOL_MODE_TOMASK, tomaskArgs); + PXOPT_ADD_MODE("-addmask", "", MAGICTOOL_MODE_ADDMASK, addmaskArgs); + PXOPT_ADD_MODE("-revertmask", "", MAGICTOOL_MODE_REVERTMASK, revertmaskArgs); + PXOPT_ADD_MODE("-mask", "", MAGICTOOL_MODE_MASK, maskArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pstamptool.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pstamptool.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pstamptool.c (revision 22161) @@ -0,0 +1,719 @@ +/* + * pstamptool.c + * + * Copyright (C) 2008 + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pxtools.h" +#include "pxdata.h" +#include "pstamptool.h" + +static bool adddatastoreMode(pxConfig *config); +static bool datastoreMode(pxConfig *config); +static bool moddatastoreMode(pxConfig *config); +static bool addReqMode(pxConfig *config); +static bool completedReqMode(pxConfig *config); +static bool listReqMode(pxConfig *config); +static bool pendingReqMode(pxConfig *config); +static bool updateReqMode(pxConfig *config); +static bool revertReqMode(pxConfig *config); +static bool addJobMode(pxConfig *config); +static bool listJobMode(pxConfig *config); +static bool pendingJobMode(pxConfig *config); +static bool updateJobMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pstamptoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(PSTAMPTOOL_MODE_ADDDATASTORE, adddatastoreMode); + MODECASE(PSTAMPTOOL_MODE_DATASTORE, datastoreMode); + MODECASE(PSTAMPTOOL_MODE_MODDATASTORE, moddatastoreMode); + MODECASE(PSTAMPTOOL_MODE_ADDREQ, addReqMode); + MODECASE(PSTAMPTOOL_MODE_COMPLETEDREQ, completedReqMode); + MODECASE(PSTAMPTOOL_MODE_LISTREQ, listReqMode); + MODECASE(PSTAMPTOOL_MODE_PENDINGREQ, pendingReqMode); + MODECASE(PSTAMPTOOL_MODE_UPDATEREQ, updateReqMode); + MODECASE(PSTAMPTOOL_MODE_REVERTREQ, revertReqMode); + MODECASE(PSTAMPTOOL_MODE_ADDJOB, addJobMode); + MODECASE(PSTAMPTOOL_MODE_LISTJOB, listJobMode); + MODECASE(PSTAMPTOOL_MODE_PENDINGJOB, pendingJobMode); + MODECASE(PSTAMPTOOL_MODE_UPDATEJOB, updateJobMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool adddatastoreMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + PXOPT_LOOKUP_STR(outProduct, config->args, "-out_product", true, false); + PXOPT_LOOKUP_STR(lastFileset, config->args, "-last_fileset", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + + if (!pstampDataStoreInsert(config->dbh, + 0, + state, + lastFileset, + outProduct, + uri + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool datastoreMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ds_id", "ds_id", "=="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("pstamptool_datastore.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampDataStore", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} +static bool moddatastoreMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(ds_id, config->args, "-ds_id", true, false); + PXOPT_LOOKUP_STR(lastFileset, config->args, "-last_fileset", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + + if (!state && !lastFileset) { + psError(PS_ERR_UNKNOWN, true, "at least one of -last_fileset or -state is required"); + return false; + } + + char *query = psStringCopy ("UPDATE pstampDataStore SET"); + bool needComma = false; + + if (lastFileset) { + psStringAppend(&query, " lastFileset = '%s'", lastFileset); + needComma = true; + } + + if (state) { + psStringAppend(&query, "%s state = '%s'", needComma ? "," : " ", state); + needComma = true; // be ready in case we add another field + } + + psStringAppend(&query, " WHERE ds_id = %" PRId64, ds_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + if (affected != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" + PRIu64 " rows were modified", affected); + return false; + } + + return true; +} + +static bool addReqMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + // PXOPT_LOOKUP_STR(outFileset, config->args, "-out_fileset", true, false); + PXOPT_LOOKUP_S64(ds_id, config->args, "-ds_id", false, false); + + char *query ="INSERT INTO pstampRequest" + " (state, uri, ds_id, fault)" + " VALUES( 'new', '%s', %" PRId64 ", 0 )"; + if (!p_psDBRunQuery(config->dbh, query, uri, ds_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + if (affected != 1) { + psError(PS_ERR_UNKNOWN, false, + "should have affected one row but %" PRIu64 " rows were modified", + affected); + return false; + } + + psS64 req_id = psDBLastInsertID(config->dbh); + + printf("%" PRId64 "\n", req_id); + + return true; +} + +static bool pendingReqMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("pstamptool_pendingreq.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampRequest"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampRequest", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool listReqMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = NULL; + psStringAppend(&query, + "SELECT * FROM pstampRequest WHERE state = 'run' AND " + "(SELECT count(*) FROM pstampJob WHERE pstampJob.req_id = pstampRequest.req_id " + " AND pstampJob.state != 'stop') = 0" ); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampRequest", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool completedReqMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = + "SELECT * FROM pstampRequest WHERE state = 'run' AND " + "(SELECT count(*) FROM pstampJob WHERE pstampJob.req_id = pstampRequest.req_id " + " AND pstampJob.state != 'stop') = 0" ; + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampRequest", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool updateReqMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + PXOPT_LOOKUP_STR(outProduct, config->args, "-outProduct", false, false); + PXOPT_LOOKUP_STR(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); + PXOPT_LOOKUP_STR(reqType, config->args, "-reqType", false, false); + + psString query = NULL; + psStringAppend(&query, "UPDATE pstampRequest SET"); + + char c = ' '; + if (state) { + psStringAppend(&query, "%c state = '%s'", c, state); + c = ','; + } + if (outProduct) { + psStringAppend(&query, "%c outProduct = '%s'", c, outProduct); + c = ','; + } + if (fault) { + psStringAppend(&query, "%c fault = %s", c, fault); + c = ','; + } + if (uri) { + psStringAppend(&query, "%c uri = '%s'", c, uri); + c = ','; + } + if (name) { + psStringAppend(&query, "%c name = '%s'", c, name); + c = ','; + } + if (reqType) { + psStringAppend(&query, "%c reqType = '%s'", c, reqType); + c = ','; + } + if (!state && !outProduct && !fault && !uri && !name && !reqType) { + psError(PS_ERR_UNKNOWN, true, "at least one of state, outProduct, fault, uri, name, and reqType must be specified"); + return false; + } + + psStringAppend(&query, " WHERE req_id = %" PRId64, req_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + // note zero is not an error + if (affected > 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" + PRIu64 " rows were modified", affected); + return false; + } + + return true; +} + +static bool revertReqMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); + + // printf("Revert request %" PRId64 "\n", req_id); + + if (!p_psDBRunQuery(config->dbh, "DELETE FROM pstampJob where req_id = %" PRId64, req_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!p_psDBRunQuery(config->dbh, + "UPDATE pstampRequest set state ='new', name=NULL, reqType=NULL, fault=0 where req_id = %" PRId64, req_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool addJobMode(pxConfig *config) +{ + bool stampJob = false; + char *query = NULL; + + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(uri, config->args, "-uri", true, false); + PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); + PXOPT_LOOKUP_STR(rownum, config->args, "-rownum", true, false); + PXOPT_LOOKUP_STR(job_type, config->args, "-job_type", false, false); + PXOPT_LOOKUP_STR(outputBase, config->args, "-outputBase", true, false); + PXOPT_LOOKUP_STR(argString, config->args, "-args", false, false); + PXOPT_LOOKUP_STR(stateString, config->args, "-state", false, false); + PXOPT_LOOKUP_STR(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + + // default value for job_type is defined in pstamptoolConfig.c + if (!strcmp(job_type, "get_image") || !strcmp(job_type, "detect_query")) { + stampJob = false; + } else if (!strcmp(job_type, "stamp")) { + stampJob = true; + } else { + psError(PS_ERR_UNKNOWN, false, "unknown value for -job_type: %s", job_type); + return false; + } + + if (stampJob && !argString) { + psError(PS_ERR_UNKNOWN, true, "-args is required for stamp job"); + return false; + } + + if (stampJob) { + query = pxDataGet("pstamptool_addjob_stampjob.sql"); + } else { + query = pxDataGet("pstamptool_addjob_otherjob.sql"); + } + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query, req_id, rownum, stateString, job_type, uri, exp_id, outputBase, fault, argString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + if (affected != 1) { + psError(PS_ERR_UNKNOWN, false, + "should have affected one row but %" PRIu64 " rows were modified", + affected); + return false; + } + + psS64 job_id = psDBLastInsertID(config->dbh); + printf("%" PRId64 "\n", job_id); + + return true; +} + +static bool listJobMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", false, false); + PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", false, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = NULL; + + if (!req_id && !job_id) { + fprintf(stderr, "either req_id or job_id must be specified\n"); + exit (1); + } + + if (req_id) { + psStringAppend(&query, + "SELECT" + " *" + " FROM pstampJob" + " WHERE req_id = %" PRId64, req_id + ); + } else { + psStringAppend(&query, + "SELECT" + " *" + " FROM pstampJob" + " WHERE job_id = %" PRId64, job_id + ); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampJob", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool pendingJobMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("pstamptool_pendingjob.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampJob"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampJob", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool updateJobMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + PXOPT_LOOKUP_STR(fault, config->args, "-fault", false, false); + + psString faultStr = NULL; + if (!fault) { + faultStr = psStringCopy(""); + } else { + psStringAppend(&faultStr, ", fault = '%s'", fault); + } + + char *query ="UPDATE pstampJob" + " SET state = '%s' %s" + " WHERE job_id = %" PRId64; + + if (!p_psDBRunQuery(config->dbh, query, state, faultStr, job_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(faultStr); + + psU64 affected = psDBAffectedRows(config->dbh); + if (affected != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" + PRIu64 " rows were modified", affected); + return false; + } + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pstamptool.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pstamptool.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pstamptool.h (revision 22161) @@ -0,0 +1,45 @@ +/* + * pstamptool.h + * + * Copyright (C) 2008 + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PSTAMPTOOL_H +#define PSTAMPTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + PSTAMPTOOL_MODE_NONE = 0x0, + PSTAMPTOOL_MODE_ADDDATASTORE, + PSTAMPTOOL_MODE_DATASTORE, + PSTAMPTOOL_MODE_MODDATASTORE, + PSTAMPTOOL_MODE_ADDREQ, + PSTAMPTOOL_MODE_LISTREQ, + PSTAMPTOOL_MODE_COMPLETEDREQ, + PSTAMPTOOL_MODE_PENDINGREQ, + PSTAMPTOOL_MODE_UPDATEREQ, + PSTAMPTOOL_MODE_REVERTREQ, + PSTAMPTOOL_MODE_ADDJOB, + PSTAMPTOOL_MODE_LISTJOB, + PSTAMPTOOL_MODE_PENDINGJOB, + PSTAMPTOOL_MODE_JOBRESULT, + PSTAMPTOOL_MODE_UPDATEJOB, +} pstamptoolMode; + +pxConfig *pstamptoolConfig(pxConfig *config, int argc, char **argv); + +#endif // PSTAMPTOOL_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pstamptoolConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pstamptoolConfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pstamptoolConfig.c (revision 22161) @@ -0,0 +1,174 @@ +/* + * pstamptoolConfig.c + * + * Copyright (C) 2008 + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pstamptool.h" + +pxConfig *pstamptoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); + psFree(config); + return NULL; + } + + // -adddatastore + psMetadata *adddatastoreArgs = psMetadataAlloc(); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-uri", 0, "define storage uri (required)", NULL); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-out_product", 0, "define output product name (required)", NULL); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-last_fileset", 0, "define last fileset seen", NULL); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-state", 0, "define datastore state", "enabled"); + + // -datastore + psMetadata *datastoreArgs = psMetadataAlloc(); + psMetadataAddS64(datastoreArgs, PS_LIST_TAIL, "-ds_id", 0, "define ds_id", 0); + psMetadataAddBool(datastoreArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -moddatastore + psMetadata *moddatastoreArgs = psMetadataAlloc(); + psMetadataAddS64(moddatastoreArgs, PS_LIST_TAIL, "-ds_id", 0, "define ds_id", 0); + psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-last_fileset", 0, "define storage uri", NULL); + psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-state", 0, "define storage uri", NULL); + + // -addreq + psMetadata *addreqArgs = psMetadataAlloc(); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0, "define request file uri", NULL); + psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-ds_id", 0, "define request ds_id", 0); + // psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-out_fileset", 0, "define request output_fileset", NULL); + + // -pendingreq + psMetadata *pendingreqArgs = psMetadataAlloc(); + psMetadataAddS64(pendingreqArgs, PS_LIST_TAIL, "-req_id", 0, "define req_id", 0); + psMetadataAddU64(pendingreqArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -listreq + psMetadata *listreqArgs = psMetadataAlloc(); + psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-req_id", 0, "define req_id", 0); + psMetadataAddU64(listreqArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -completedreq + psMetadata *completedreqArgs = psMetadataAlloc(); + psMetadataAddU64(completedreqArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(completedreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updatereq + psMetadata *updatereqArgs = psMetadataAlloc(); + psMetadataAddS64(updatereqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id for which to change state", 0); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-outProduct", 0, "define request outProduct", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-fault", 0, "define request fault code", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-uri", 0, "define the uri", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0, "define the name", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-reqType", 0, "define the reqType", NULL); + + // -revertreq + psMetadata *revertreqArgs = psMetadataAlloc(); + psMetadataAddS64(revertreqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id for which to revert", 0); + + // -addjob + psMetadata *addjobArgs = psMetadataAlloc(); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-uri", 0, "define job file uri", NULL); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-req_id", 0, "define job req_id", 0); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-rownum", 0, "define job rownum", NULL); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-job_type", 0, "define job job_type", "stamp"); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-outputBase", 0, "define job outputBase", NULL); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-args", 0, "define job args", NULL); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-state", 0, "new state", "run"); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0, "exposure id", 0); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-fault", 0, "new result", NULL); + + // -listjob + psMetadata *listjobArgs = psMetadataAlloc(); + psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-req_id", 0, "define request", 0); + psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-job_id", 0, "define job", 0); + psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listjobArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -pendingjob + psMetadata *pendingjobArgs = psMetadataAlloc(); + psMetadataAddS64(pendingjobArgs, PS_LIST_TAIL, "-job_id", 0, "define job", 0); + psMetadataAddS64(pendingjobArgs, PS_LIST_TAIL, "-req_id", 0, "define request", 0); + psMetadataAddU64(pendingjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingjobArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updatejob + psMetadata *updatejobArgs = psMetadataAlloc(); + psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-job_id", 0, "req_id for which to change state", 0); + psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); + // psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-result", 0, "new result", NULL); + psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-fault", 0, "new result", NULL); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-addreq", "", PSTAMPTOOL_MODE_ADDREQ, addreqArgs); + PXOPT_ADD_MODE("-pendingreq", "", PSTAMPTOOL_MODE_PENDINGREQ, pendingreqArgs); + PXOPT_ADD_MODE("-updatereq", "", PSTAMPTOOL_MODE_UPDATEREQ, updatereqArgs); + PXOPT_ADD_MODE("-listreq", "", PSTAMPTOOL_MODE_LISTREQ, listreqArgs); + PXOPT_ADD_MODE("-completedreq", "", PSTAMPTOOL_MODE_COMPLETEDREQ, completedreqArgs); + PXOPT_ADD_MODE("-revertreq", "", PSTAMPTOOL_MODE_REVERTREQ, revertreqArgs); + + PXOPT_ADD_MODE("-addjob", "", PSTAMPTOOL_MODE_ADDJOB, addjobArgs); + PXOPT_ADD_MODE("-listjob", "", PSTAMPTOOL_MODE_LISTJOB, listjobArgs); + PXOPT_ADD_MODE("-pendingjob", "", PSTAMPTOOL_MODE_PENDINGJOB, pendingjobArgs); + PXOPT_ADD_MODE("-updatejob", "", PSTAMPTOOL_MODE_UPDATEJOB, updatejobArgs); + + PXOPT_ADD_MODE("-adddatastore", "", PSTAMPTOOL_MODE_ADDDATASTORE, adddatastoreArgs); + PXOPT_ADD_MODE("-datastore", "", PSTAMPTOOL_MODE_DATASTORE, datastoreArgs); + PXOPT_ADD_MODE("-moddatastore", "", PSTAMPTOOL_MODE_MODDATASTORE, moddatastoreArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxadmin.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxadmin.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxadmin.c (revision 22161) @@ -0,0 +1,221 @@ +/* + * pxadmin.c + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "pxtools.h" +#include "pxadmin.h" + +bool createMode(pxConfig *config); +bool deleteMode(pxConfig *config); +static bool runMultipleStatments(pxConfig *config, const char *query); + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pxAdminConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + case PXADMIN_MODE_RECREATE: + if (!deleteMode(config)) { + goto FAIL; + } + // fall through + case PXADMIN_MODE_CREATE: + if (!createMode(config)) { + goto FAIL; + } + break; + case PXADMIN_MODE_DELETE: + if (!deleteMode(config)) { + goto FAIL; + } + break; + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +bool createMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("pxadmin_create_tables.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // BEGIN + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!runMultipleStatments(config, query)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + // COMMIT + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +bool deleteMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + { + char line[128], answer[128]; + + psMetadataItem *name = pmConfigUserSite(config->modules, "DBNAME", PS_DATA_STRING); + if (!name) { + psError(PS_ERR_UNKNOWN, false, "Unable to determine database name."); + return false; + } + psString dbName = name->data.str; + + fprintf(stdout, "*** delete the tables from database %s? ***\n", dbName); + fprintf(stdout, "*** to delete the tables, answer YES, and give password ***\n"); + fprintf(stdout, "*** WARNING: this action is permanent ***\n\n"); + + fprintf(stdout, "delete the tables (YES/[n]): "); + if (!fgets(line, 128, stdin)) { + psError(PS_ERR_IO, true, "Unable to read response."); + return false; + } + sscanf(line, "%s", answer); + if (strcmp (answer, "YES")) { + psError(PS_ERR_UNKNOWN, true, "tables NOT deleleted"); + return false; + } + + fprintf(stdout, "enter dbh connection password: "); + if (!fgets(line, 128, stdin)) { + psError(PS_ERR_IO, true, "Unable to read response."); + return false; + } + sscanf(line, "%s", answer); + + psMetadataItem *pass = pmConfigUserSite(config->modules, "DBPASSWORD", PS_DATA_STRING); + if (!pass) { + psError(PS_ERR_UNKNOWN, false, "Unable to determine database password."); + return false; + } + psString dbPassword = pass->data.str; + if (strcmp (answer, dbPassword)) { + psError(PS_ERR_UNKNOWN, true, "invalid passwd - tables NOT deleleted"); + return false; + } + } + + psString query = pxDataGet("pxadmin_drop_tables.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // BEGIN + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!runMultipleStatments(config, query)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + // COMMIT + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool runMultipleStatments(pxConfig *config, const char *query) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(query, false); + + // loop over all statements in string + psList *statements = psStringSplit(query, ";", false); + + psString q = NULL; + psListIterator *iter = psListIteratorAlloc(statements, PS_LIST_HEAD, false); + while ((q = psListGetAndIncrement(iter))) { + if (!p_psDBRunQuery(config->dbh, q)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(iter); + psFree(statements); + return false; + } + } + psFree(iter); + psFree(statements); + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxadmin.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxadmin.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxadmin.h (revision 22161) @@ -0,0 +1,34 @@ +/* + * pxadmin.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXADMIN_H +#define PXADMIN_H 1 + +#include "pxtools.h" + +typedef enum { + PXADMIN_MODE_NONE = 0x0, + PXADMIN_MODE_CREATE, + PXADMIN_MODE_DELETE, + PXADMIN_MODE_RECREATE +} pxAdminMode; + +pxConfig *pxAdminConfig (pxConfig *config, int argc, char **argv); + +#endif // PXADMIN_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxadminConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxadminConfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxadminConfig.c (revision 22161) @@ -0,0 +1,153 @@ +/* + * pxadminConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pxadmin.h" + +static void pxAdminShowDB (const pxConfig *config, const char *program) +{ + fprintf (stderr, "\nPan-STARRS DataBase Admin Tool\n\n"); + fprintf (stderr, "Usage: %s [mode]\n", program); + fprintf (stderr, " [mode] : -create | -delete\n\n"); + + psMetadataItem *server = pmConfigUserSite(config->modules, "DBSERVER", PS_DATA_STRING); + psMetadataItem *user = pmConfigUserSite(config->modules, "DBUSER", PS_DATA_STRING); + psMetadataItem *name = pmConfigUserSite(config->modules, "DBNAME", PS_DATA_STRING); + psMetadataItem *port = pmConfigUserSite(config->modules, "DBPORT", PS_TYPE_S32); + + if (!server || !user || !name) { + psErrorClear(); + psWarning("Unable to determine database connection details."); + return; + } + if (!port) { + psErrorClear(); + } + + fprintf (stderr, "connecting to %s as %s (port %d)\n", server->data.str, user->data.str, + port ? port->data.S32 : 0); + fprintf (stderr, "using database %s\n\n", name->data.str); + + return; +} + +pxConfig *pxAdminConfig(pxConfig *config, int argc, char **argv) +{ + if (!psArgumentGet(argc, argv, "-dbname")) { + psError(PS_ERR_UNEXPECTED_NULL, true, + "Command-line argument '-dbname XXX' is required, for database safety"); + return NULL; + } + + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // Parse other command-line arguments + psMetadata *arguments = psMetadataAlloc(); // The arguments, with default values + + int N; + config->mode = PXADMIN_MODE_NONE; + if ((N = psArgumentGet(argc, argv, "-create"))) { + psArgumentRemove(N, &argc, argv); + if (config->mode) { + psAbort("only one mode selection is allowed"); + } + config->mode = PXADMIN_MODE_CREATE; + } + if ((N = psArgumentGet(argc, argv, "-delete"))) { + psArgumentRemove(N, &argc, argv); + if (config->mode) { + psAbort("only one mode selection is allowed"); + } + config->mode = PXADMIN_MODE_DELETE; + } + if ((N = psArgumentGet(argc, argv, "-recreate"))) { + psArgumentRemove(N, &argc, argv); + if (config->mode) { + psAbort("only one mode selection is allowed"); + } + config->mode = PXADMIN_MODE_RECREATE; + } + + // paul's argument parsing convention requires: -key value + psMetadataAddStr(arguments, PS_LIST_TAIL, "-create", 0, + "create all IPP tables", ""); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-delete", 0, + "delete all IPP tables", ""); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-recreate", 0, + "delete and recreate all tables", ""); + + if (config->mode == PXADMIN_MODE_NONE) { + pxAdminShowDB (config, argv[0]); + fprintf (stderr, "admin mode not specified\n"); + + psArgumentHelp(arguments); + psFree(arguments); + psFree(config); + return NULL; + } + + if ((N = psArgumentGet (argc, argv, "-help"))) { + pxAdminShowDB (config, argv[0]); + psArgumentHelp(arguments); + psFree(arguments); + psFree(config); + return NULL; + } + + if (! psArgumentParse(arguments, &argc, argv) || argc != 1) { + pxAdminShowDB (config, argv[0]); + psArgumentHelp(arguments); + psFree(arguments); + psFree(config); + return NULL; + } + + psFree(arguments); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + // save argv/argc + config->argv = argv; + config->argc = argc; + + return config; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxcam.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxcam.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxcam.c (revision 22161) @@ -0,0 +1,268 @@ +/* + * pxcam.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxcam.h" + +bool pxcamSetSearchArgs (psMetadata *md) { + + psMetadataAddS64(md, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); + psMetadataAddS64(md, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min", 0, "define min", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max", 0, "define max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min", 0, "define min", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max", 0, "define max", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min", 0, "define min", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max", 0, "define max", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define min fraction of saturated pixels", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min", 0, "define max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max", 0, "define max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min", 0, "define max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min", 0, "define min", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max", 0, "define max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-az_min", 0, "define min", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-az_max", 0, "define max", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(md, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + + return true; +} + +bool pxcamGetSearchArgs (pxConfig *config, psMetadata *where) { + + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + + return true; +} + +bool pxcamRunSetState(pxConfig *config, psS64 cam_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid camRun state: %s", state); + return false; + } + + char *query = "UPDATE camRun SET state = '%s' WHERE cam_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, cam_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for cam_id %" PRId64, cam_id); + return false; + } + + return true; +} + + +bool pxcamRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid chipRun state: %s", state); + return false; + } + + psString query = psStringCopy("UPDATE camRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET state = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxcamRunSetLabel(pxConfig *config, psS64 cam_id, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + char *query = "UPDATE camRun SET camRun.label = '%s' WHERE cam_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, label, cam_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for cam_id %" PRId64, cam_id); + return false; + } + + return true; +} + +bool pxcamRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + psString query = psStringCopy("UPDATE camRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET camRun.label = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxcamQueueByChipID(pxConfig *config, + psS64 chip_id, + char *workdir, + char *label, + char *recipe, + char *expgroup, + char *dvodb, + char *tess_id, + char *end_stage) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("camtool_queue_chip_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + } + + // queue the exp + // XXX chip_id is being cast here work around psS64 have a different type + // different on 32/64 + if (!p_psDBRunQuery(config->dbh, query, + "new", // state + workdir ? workdir : "NULL", + "dirty", //workdir_state + label ? label : "NULL", + recipe ? recipe : "NULL", + expgroup ? expgroup : "NULL", + dvodb ? dvodb : "NULL", + tess_id ? tess_id : "NULL", + end_stage ? end_stage : "NULL", + (long long)chip_id + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row"); + return false; + } + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxcam.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxcam.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxcam.h (revision 22161) @@ -0,0 +1,45 @@ +/* + * pxcam.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXCAM_H +#define PXCAM_H 1 + +#include + +#include "pxtools.h" + +bool pxcamRunSetState(pxConfig *config, psS64 cam_id, const char *state); +bool pxcamRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); +bool pxcamRunSetLabel(pxConfig *config, psS64 cam_id, const char *label); +bool pxcamRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); + +bool pxcamSetSearchArgs (psMetadata *md); +bool pxcamGetSearchArgs (pxConfig *config, psMetadata *where); + +bool pxcamQueueByChipID(pxConfig *config, + psS64 chip_id, + char *workdir, + char *label, + char *recipe, + char *expgroup, + char *dvodb, + char *tess_id, + char *end_stage); + +#endif // PXCAM_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxchip.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxchip.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxchip.c (revision 22161) @@ -0,0 +1,255 @@ +/* + * pxchip.c + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxchip.h" + +bool pxchipSetSearchArgs (psMetadata *md) { + + // XXX need to allow multiple exp_ids + psMetadataAddS64(md, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + // psMetadataAddStr(md, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + // psMetadataAddStr(md, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(md, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min", 0, "search by min", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min", 0, "search by min", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max", 0, "search by max", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min", 0, "search by min", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max", 0, "search by max", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by max fraction of saturated pixels", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by min fraction of saturated pixels", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min", 0, "search by min", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-az_min", 0, "search by min", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-az_max", 0, "search by max", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); + psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(md, PS_LIST_TAIL, "-solang_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-solang_max", 0, "search by max solar angle", NAN); + psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + return true; +} + +// XXX explicit refs to rawExp and chipRun? +// XXX careful with entries that could match rawExp rawImfile, chipRun, or chipProcessedImfile (eg, bg) +bool pxchipGetSearchArgs (pxConfig *config, psMetadata *where) { + + // definebyquery : rawExp only + // updaterun : rawExp, chipRun + // pendingimfile : rawExp, chipRun + // processedimfile : rawExp, chipRun, chipProcessedImfile + // revertprocessedimfile : rawExp, chipProcessedImfile + // updateprocessedimfile : chipProcessedImfile + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "rawExp.solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "rawExp.solang", "<"); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + return true; +} + +bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid chipRun state: %s", state); + return false; + } + + char *query = "UPDATE chipRun SET state = '%s' WHERE chip_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, chip_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for chip_id %" PRId64, chip_id); + return false; + } + + return true; +} + + +bool pxchipRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid chipRun state: %s", state); + return false; + } + + + psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) SET state = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxchipRunSetLabel(pxConfig *config, psS64 chip_id, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + char *query = "UPDATE chipRun SET label = '%s' WHERE chip_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, label, chip_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for chip_id %" PRId64, chip_id); + return false; + } + + return true; +} + + +bool pxchipRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) SET label = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +psS64 pxchipQueueByExpTag(pxConfig *config, + psS64 exp_id, + const char *workdir, + const char *label, + const char *reduction, + const char *expgroup, + const char *dvodb, + const char *tess_id, + const char *end_stage) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // create a chipRun + if (!chipRunInsert(config->dbh, + 0x0, // chip_id + exp_id, + "new", // state + workdir, + "dirty", // workdir_state + label, + reduction, + expgroup, + dvodb, + tess_id, + end_stage) + ) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return 0; + } + + return psDBLastInsertID(config->dbh); +} + Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxchip.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxchip.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxchip.h (revision 22161) @@ -0,0 +1,46 @@ +/* + * pxchip.h + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXCHIP_H +#define PXCHIP_H 1 + +#include + +#include "pxtools.h" + +bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state); +bool pxchipRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); +bool pxchipRunSetLabel(pxConfig *config, psS64 chip_id, const char *label); +bool pxchipRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); + +psS64 pxchipQueueByExpTag(pxConfig *config, + psS64 exp_id, + const char *workdir, + const char *label, + const char *reduction, + const char *expgroup, + const char *dvodb, + const char *tess_id, + const char *end_stage); + + +bool pxchipSetSearchArgs (psMetadata *md); +bool pxchipGetSearchArgs (pxConfig *config, psMetadata *where); + +#endif // PXCHIP_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxconfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxconfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxconfig.c (revision 22161) @@ -0,0 +1,227 @@ +/* + * pxconfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include +#include + +#include "pxtools.h" + +static void pxConfigFree(pxConfig *ptr); + +pxConfig *pxConfigAlloc(void) +{ + pxConfig *config; + + config = psAlloc(sizeof(pxConfig)); + psMemSetDeallocator(config, (psFreeFunc)pxConfigFree); + + config->modeName = NULL; + config->mode = 0; + config->dbh = NULL; + config->modules = NULL; + // XXX config->where = NULL; + config->args = NULL; + + return config; +} + +static void pxConfigFree(pxConfig *config) +{ + psFree(config->modeName); + psFree(config->dbh); + psFree(config->modules); + // XXX psFree(config->where); + psFree(config->args); +} + +void pxUsage(FILE *stream, int argc, char **argv, const char *modeName, psMetadata *argSet) +{ + fprintf(stream, "Usage: %s %s []\n\n", argv[0], modeName); + fprintf(stream, "%s:\n", modeName); + + psArgumentHelpSimple(stream, argSet); +} + +bool pxGetOptions(FILE *stream, int argc, char **argv, pxConfig *config, psMetadata *modes, psMetadata *argSets) +{ + // figure out what mode we're running in + psMetadataIterator *iter = psMetadataIteratorAlloc(modes, PS_LIST_HEAD, NULL); + psMetadataItem *item = NULL; + while ((item = psMetadataGetAndIncrement(iter))) { + int N = 0; + if ((N = psArgumentGet(argc, argv, item->name))) { + psArgumentRemove(N, &argc, argv); + // check for duplicate mode specification + if (config->mode) { + psError(PS_ERR_UNKNOWN, true, "only one mode selection is allowed"); + fprintf(stream, "only one mode selection is allowed\n"); + pxUsage(stream, argc, argv, "", modes); + psFree(iter); + return NULL; + } + + config->mode = item->data.U32; + config->modeName = psStringCopy(item->name); + + bool status = false; + psMetadata *argset = psMetadataLookupMetadata(&status, argSets, item->name); + // make sure we can find the argSet for the sepcified mode + if (!status) { + psError(PS_ERR_UNKNOWN, true, "can not find arguments for mode"); + fprintf(stderr, "can not find arugments for mode"); + psFree(iter); + return NULL; + } + + config->args = psMemIncrRefCounter(argset); + } + + } + + psFree(iter); + + // make sure we found a mode + if (config->mode == PXTOOL_MODE_NONE) { + psError(PS_ERR_UNKNOWN, true, "mode argument is required"); + fprintf(stderr, "mode argument is required\n"); + pxUsage(stream, argc, argv, "", modes); + return NULL; + } + + // actually parse the command line + if (!psArgumentParse(config->args, &argc, argv)) { + psError(PS_ERR_UNKNOWN, false, "error parsing arguments"); + fprintf(stderr, "error parsing arguments\n"); + pxUsage(stream, argc, argv, config->modeName, config->args); + return NULL; + } + + // look for left overs on the command line + if (argc != 1) { + psError(PS_ERR_UNKNOWN, false, "extra arguments: "); + fprintf(stderr, "extra arguments: "); + for (int i = 1; i < argc; i++) { + fprintf (stderr, "%s ", argv[i]); + } + fprintf(stderr, "\n"); + pxUsage(stream, argc, argv, config->modeName, config->args); + return NULL; + } + + // make sure that all required parameters have been specified + iter = psMetadataIteratorAlloc(config->args, PS_LIST_HEAD, NULL); + item = NULL; // Item from iterator + while ((item = psMetadataGetAndIncrement(iter))) { + if (strstr(item->comment, "require") == NULL) { + continue; + } + if (strstr(item->comment, "(found)") != NULL) { + continue; + } + + switch (item->type) { + case PS_DATA_BOOL: + psError(PS_ERR_UNKNOWN, false, "boolean type can not be required"); + fprintf(stderr, "boolean type can not be required\n"); + psFree(iter); + return NULL; + break; + case PS_DATA_S8: + if (item->data.S8 != INT8_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_S16: + if (item->data.S16 != INT16_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_S32: + if (item->data.S32 != INT32_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_S64: + if (item->data.S64 != INT64_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_U8: + if (item->data.U8 != UINT8_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_U16: + if (item->data.U16 != UINT16_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_U32: + if (item->data.U32 != UINT32_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_U64: + if (item->data.U64 != UINT64_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_F32: + case PS_DATA_F64: + if (!isnan(item->data.F64)) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_STRING: + case PS_DATA_TIME: + case PS_DATA_METADATA: + if (item->data.V) { + break; + } + goto ARG_REQUIRED; + default: + psError(PS_ERR_UNKNOWN, false, "unknown argument type"); + fprintf(stream, "unknown argument type\n"); + psFree(iter); + return NULL; + ARG_REQUIRED: + psError(PS_ERR_UNKNOWN, false, "argument %s is required", item->name); + fprintf(stream, "argument %s is required\n", item->name); + pxUsage(stream, argc, argv, config->modeName, config->args); + psFree(iter); + return NULL; + } + + } + + psFree(iter); + + // save argv/argc + config->argv = argv; + config->argc = argc; + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxconfig.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxconfig.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxconfig.h (revision 22161) @@ -0,0 +1,39 @@ +/* + * pxconfig.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXCONFIG_H +#define PXCONFIG_H 1 + +#include "pslib.h" +#include "psmodules.h" + +typedef struct { + char *modeName; + int mode; + pmConfig *modules; + psDB *dbh; + psMetadata *args; + // XXX deprecate psMetadata *where; + int argc; + char **argv; +} pxConfig; + +pxConfig *pxConfigAlloc(void); + +#endif // PXCONFIG_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxdata.c.template =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxdata.c.template (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxdata.c.template (revision 22161) @@ -0,0 +1,145 @@ +/* + * pxdata.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include // getenv +#include +#include +#include + +#include + +#include "pxtools.h" +#include "pxdata.h" + +static psHash *cachedData = NULL; + +static int test_f(const char *path); + +psString pxDataGet(const char *filename) +{ + PS_ASSERT_PTR_NON_NULL(filename, NULL); + + { + psString filepath = NULL; + + // look to see if PXDATA is set + char *PXDATA = getenv("PXDATA"); + if (PXDATA) { + // if it is, then prepend filename to PXDATA + psStringAppend(&filepath, "%s/%s", PXDATA, filename); + } + + // see if we have a valid filename + if (test_f(filepath)) { + // if we do, slurp the file + psString data = psSlurpFilename(filepath); + if (!data) { + psError(PS_ERR_IO, false, "failed to slurp %s", filepath); + psFree(filepath); + return NULL; + } + psTrace("pxdata", PS_LOG_INFO, "slupred %s", filepath); + psFree(filepath); + + return data; + } + + psFree(filepath); + } + + { + psString filepath = NULL; + + // look under our share path + psStringAppend(&filepath, "%s/%s", "@PKGDATADIR@", filename); + + // see if we have a valid filename + if (test_f(filepath)) { + // if we do, slurp the file + psString data = psSlurpFilename(filepath); + if (!data) { + psError(PS_ERR_IO, false, "failed to slurp %s", filepath); + psFree(filepath); + return NULL; + } + psTrace("pxdata", PS_LOG_INFO, "slupred %s", filepath); + psFree(filepath); + + return data; + } + + psFree(filepath); + } + + // couldn't find a matching filename + psError(PS_ERR_IO, true, "can't find a file in search path(s) to match : %s", filename); + + return NULL; +} + +static int test_f(const char *path) +{ + // copied from coreutils 6.4 test.c licenced under GPL v2 + struct stat stat_buf; + return (stat (path, &stat_buf) == 0 + && S_ISREG (stat_buf.st_mode)); +} + +psString pxDataGetCached(const char *filename) +{ + bool persistence = p_psMemAllocatePersistent(true); + + // make sure the cache is initalized + if (!cachedData) { + cachedData = psHashAlloc(10); + } + + // look to see if we've cached this file + { + psString data = psHashLookup(cachedData, filename); + if (data) { + p_psMemAllocatePersistent(persistence); + return psStringCopy(data); + } + } + + // we didn't have a cached copy so we need to try to read the file + psString data = pxDataGet(filename); + if (!data) { + p_psMemAllocatePersistent(persistence); + psError(PS_ERR_IO, false, "pxDataGet(%s) failed", filename); + return NULL; + } + + // store a copy of the file + if (!psHashAdd(cachedData, filename, data)) { + psFree(data); + p_psMemAllocatePersistent(persistence); + psError(PS_ERR_UNKNOWN, false, "psHashAdd() failed"); + return NULL; + } + + p_psMemAllocatePersistent(persistence); + + return psStringCopy(data); +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxdata.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxdata.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxdata.h (revision 22161) @@ -0,0 +1,28 @@ +/* + * pxdata.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXDATA_H +#define PXDATA_H 1 + +#include + +psString pxDataGet(const char *filename); +psString pxDataGetCached(const char *filename); + +#endif // PXDATA_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxerrors.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxerrors.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxerrors.c (revision 22161) @@ -0,0 +1,42 @@ +/* + * pxerrors.c + * + * Copyright (C) 2006 Eugene Magnier + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "pxtools.h" + +psExit pxerrorGetExitStatus () { + + psErrorCode err = psErrorCodeLast (); + switch (err) { + case PXTOOLS_ERR_SYS: + return PS_EXIT_SYS_ERROR; + case PXTOOLS_ERR_CONFIG: + return PS_EXIT_CONFIG_ERROR; + case PXTOOLS_ERR_PROG: + return PS_EXIT_PROG_ERROR; + case PXTOOLS_ERR_DATA: + return PS_EXIT_DATA_ERROR; + default: + return PS_EXIT_UNKNOWN_ERROR; + } + return PS_EXIT_UNKNOWN_ERROR; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxfake.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxfake.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxfake.c (revision 22161) @@ -0,0 +1,178 @@ +/* + * pxfake.c + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxcam.h" + +bool pxfakeRunSetState(pxConfig *config, psS64 fake_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid fakeRun state: %s", state); + return false; + } + + char *query = "UPDATE fakeRun SET state = '%s' WHERE fake_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, fake_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for fake_id %" PRId64, fake_id); + return false; + } + + return true; +} + + +bool pxfakeRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid fakeRun state: %s", state); + return false; + } + + psString query = psStringCopy("UPDATE fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET state = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxfakeRunSetLabel(pxConfig *config, psS64 fake_id, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + char *query = "UPDATE fakeRun SET fakeRun.label = '%s' WHERE fake_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, label, fake_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for fake_id %" PRId64, fake_id); + return false; + } + + return true; +} + + +bool pxfakeRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + psString query = psStringCopy("UPDATE fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET fakeRun.label = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +psS64 pxfakeQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *label, + char *reduction, + char *expgroup, + char *dvodb, + char *tess_id, + char *end_stage) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psTrace("pxtool", PS_LOG_INFO, "attemping to queue cam_id: %"PRId64, cam_id); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("faketool_queue_cam_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + } + + // queue the exp + // XXX chip_id is being cast here work around psS64 have a different type + // different on 32/64 + if (!p_psDBRunQuery(config->dbh, query, + "new", // state + workdir ? workdir : "NULL", + label ? label : "NULL", + reduction? reduction: "NULL", + expgroup ? expgroup : "NULL", + dvodb ? dvodb : "NULL", + tess_id ? tess_id : "NULL", + end_stage ? end_stage : "NULL", + (long long)cam_id + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) != 1) { + psError(PS_ERR_UNKNOWN, false, + "should have changed just one row"); + return false; + } + + return psDBLastInsertID(config->dbh); +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxfake.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxfake.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxfake.h (revision 22161) @@ -0,0 +1,44 @@ +/* + * pxfake.h + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXFAKE_H +#define PXFAKE_H 1 + +#include + +#include "pxtools.h" + +bool pxfakeRunSetState(pxConfig *config, psS64 fake_id, const char *state); +bool pxfakeRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); +bool pxfakeRunSetLabel(pxConfig *config, psS64 fake_id, const char *label); +bool pxfakeRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); + + +psS64 pxfakeQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *label, + char *reduction, + char *expgroup, + char *dvodb, + char *tess_id, + char *end_stage); + + +#endif // PXFAKE_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxfault.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxfault.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxfault.c (revision 22161) @@ -0,0 +1,65 @@ +/* + * pxfault.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" + +bool pxSetFaultCode(psDB *dbh, const char *tableName, psMetadata *where, psS16 code) +{ + PS_ASSERT_PTR_NON_NULL(dbh, false); + PS_ASSERT_PTR_NON_NULL(tableName, false); + PS_ASSERT_PTR_NON_NULL(where, false); + +#if 0 + // map code string to numeric fault code + psU32 code = mapCodeStrToInt(codeStr); + if (code == (psU32)-1) { + psError(PS_ERR_UNKNOWN, false, "error resolving error code"); + return false; + } +#endif + + // update the database + psMetadata *values = psMetadataAlloc(); + if (!psMetadataAddS16(values, PS_LIST_HEAD, "fault", 0, NULL, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to add metadata item fault"); + psFree(values); + return false; + } + + long rowsAffected = psDBUpdateRows(dbh, tableName, where, values); + psFree(values); + if (rowsAffected < 0) { + // database error + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (rowsAffected < 1) { + // we didn't do anything + psError(PS_ERR_UNKNOWN, false, "zero rows were affected - either the search criteria didn't match any rows or the field already had the value being set."); + return false; + } + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxinject.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxinject.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxinject.c (revision 22161) @@ -0,0 +1,163 @@ +/* + * pxinject.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxinject.h" + +static bool newExpMode(pxConfig *config); +static bool newImfileMode(pxConfig *config); +static bool updatenewExpMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pxinjectConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(PXINJECT_MODE_NEWEXP, newExpMode); + MODECASE(PXINJECT_MODE_NEWIMFILE, newImfileMode); + MODECASE(PXINJECT_MODE_UPDATENEWEXP, updatenewExpMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint (stderr, "failure\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool newExpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(tmp_exp_name, config->args, "-tmp_exp_name", true, false); + PXOPT_LOOKUP_STR(tmp_camera, config->args, "-tmp_inst", true, false); + PXOPT_LOOKUP_STR(tmp_telescope, config->args, "-tmp_telescope", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (!newExpInsert(config->dbh, + 0, // exp_id + tmp_exp_name, + tmp_camera, + tmp_telescope, + "reg", // state + workdir, + "dirty", + reduction, + dvodb, + tess_id, + end_stage, + label, + NULL // epoch + ) + ) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psS64 exp_id = psDBLastInsertID(config->dbh); + + psMetadata *md = psMetadataAlloc(); + if (!psMetadataAddS64(md, PS_LIST_TAIL, "exp_id", 0, NULL, exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(md); + } + + // negate simple so the default is true + if (!ippdbPrintMetadata(stdout, md, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(md); + return false; + } + + psFree(md); + + return true; +} + +static bool newImfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(tmp_class_id, config->args, "-tmp_class_id", true, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + + // insert with error flag state set to 0 (no errors) + if (!newImfileInsert(config->dbh, exp_id, tmp_class_id, uri, NULL)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updatenewExpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + + if (state) { + // set detRun.state to state + return pxnewExpSetState(config, exp_id, state); + } + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxinject.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxinject.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxinject.h (revision 22161) @@ -0,0 +1,34 @@ +/* + * pxinject.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXINJECT_H +#define PXINJECT_H 1 + +#include "pxtools.h" + +typedef enum { + PXINJECT_MODE_NONE = 0x0, + PXINJECT_MODE_NEWEXP, + PXINJECT_MODE_NEWIMFILE, + PXINJECT_MODE_UPDATENEWEXP +} pxinjectMode; + +pxConfig *pxinjectConfig(pxConfig *config, int argc, char **argv); + +#endif // PXINJECT_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxinjectConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxinjectConfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxinjectConfig.c (revision 22161) @@ -0,0 +1,98 @@ +/* + * pxinjectConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pxinject.h" + +pxConfig *pxinjectConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -newExp + psMetadata *newExpArgs = psMetadataAlloc(); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-tmp_exp_name", 0, "define the exp_name (required)", NULL); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-tmp_inst", 0, "define the camera name (required)", NULL); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-tmp_telescope", 0, "define the telescope name (required)", NULL); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", 0); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-dvodb", 0, "define the dvodb for the next processing step", NULL); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-tess_id", 0, "define the tess_id for the next processing step", NULL); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-end_stage", 0, "define the end goal processing step", NULL); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-label", 0, "define a label (carried to chip stage)", NULL); + psMetadataAddBool(newExpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -newImfile + psMetadata *newImfileArgs = psMetadataAlloc(); + psMetadataAddS64(newImfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define the exp_id (required)", 0); + psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-tmp_class_id", 0, "define the class ID (required)", NULL); + psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-uri", 0, "define the URI (required)", NULL); + + // -updatenewExp + psMetadata *updatenewExpArgs = psMetadataAlloc(); + psMetadataAddS64(updatenewExpArgs, PS_LIST_TAIL, "-exp_id", 0, "define the exp_id (required)", 0); + psMetadataAddStr(updatenewExpArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-newExp", "", PXINJECT_MODE_NEWEXP, newExpArgs); + PXOPT_ADD_MODE("-newImfile", "", PXINJECT_MODE_NEWIMFILE, newImfileArgs); + PXOPT_ADD_MODE("-updatenewExp", "", PXINJECT_MODE_UPDATENEWEXP, updatenewExpArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxregister.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxregister.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxregister.c (revision 22161) @@ -0,0 +1,59 @@ +/* + * register.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxregister.h" + +bool pxnewExpSetState(pxConfig *config, psS64 exp_id, const char *state) +{ + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "0 is not a valid exp_id"); + return false; + } + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "reg", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid newExp state: %s", state); + return false; + } + + char *query = "UPDATE newExp SET state = '%s' WHERE exp_id = %"PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, exp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for exp_id %"PRId64, exp_id); + return false; + } + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxregister.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxregister.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxregister.h (revision 22161) @@ -0,0 +1,29 @@ +/* + * pxregister.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXREGISTER_H +#define PXREGISTER_H 1 + +#include + +#include "pxtools.h" + +bool pxnewExpSetState(pxConfig *config, psS64 exp_id, const char *state); + +#endif // PXREGISTER_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtag.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtag.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtag.c (revision 22161) @@ -0,0 +1,102 @@ +/* + * pxtag.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pxtag.h" + +psString pxGenExpTag(pxConfig *config, const char *exp_name) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // start a transaction so we don't increment the expTag counter unless we + // can successfully retreive it's value + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return NULL; + } + + if (!p_psDBRunQuery(config->dbh, + "UPDATE expTagCounter SET counter = LAST_INSERT_ID(counter + 1)")) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return NULL; + } + + // psDBLastInsertID() can't be used here as it called mysql_insert_id() + // which doesn't work with this trick. See: + // http://dev.mysql.com/doc/refman/4.1/en/information-functions.html + if (!p_psDBRunQuery(config->dbh, "SELECT LAST_INSERT_ID() as counter")) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return NULL; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return NULL; + } + // sanity check that we only got one row + if (psArrayLength(output) != 1) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "should have gotten 1 row but %lu rows were returned", psArrayLength(output)); + psFree(output); + return NULL; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return NULL; + } + + psMetadata *row = output->data[0]; + bool status = false; + psU64 counter = psMetadataLookupU64(&status, row, "counter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for counter"); + psFree(output); + return NULL; + } + psString exp_id = NULL; + psStringAppend(&exp_id, "%s.%" PRIu64, exp_name, counter); + psFree(output); + + return exp_id; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtag.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtag.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtag.h (revision 22161) @@ -0,0 +1,29 @@ +/* + * pxtag.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXTAG_H +#define PXTAG_H 1 + +#include + +#include "pxconfig.h" + +psString pxGenExpTag(pxConfig *config, const char *exp_name); + +#endif // PXTAG_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtools.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtools.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtools.c (revision 22161) @@ -0,0 +1,46 @@ +/* + * pxtool.c + * + * Copyright (C) 2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" + +bool pxIsValidState(const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // XXX replace strncmp with strcmp + + if (!((strncmp(state, "new", 4) == 0) + || (strncmp(state, "reg", 3) == 0) + || (strncmp(state, "full", 5) == 0) + || (strncmp(state, "goto_cleaned", 8) == 0) + || (strncmp(state, "cleaned", 8) == 0) + || (strncmp(state, "update", 7) == 0) + || (strncmp(state, "purged", 7) == 0) + || (strncmp(state, "goto_purged", 12) == 0))) { + return false; + } + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtools.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtools.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtools.h (revision 22161) @@ -0,0 +1,480 @@ +/* + * pxtools.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXTOOLS_H +#define PXTOOLS_H 1 + +#include +#include // for strcmp and strncmp +#include // for strcasecmp +#include // for unlink + +// #include +// #include +// #include + +#include +#include +#include + +#include "pxconfig.h" +#include "pxtoolsErrorCodes.h" + +#include "pxcam.h" +#include "pxchip.h" +#include "pxdata.h" +#include "pxfake.h" +#include "pxwarp.h" +#include "pxregister.h" +#include "pxtag.h" +#include "pxtree.h" + +# define MAX_ROWS 10e9 +# define PXTOOL_MODE_NONE 0x0 + +bool pxIsValidState(const char *state); + +bool pxSetFaultCode(psDB *dbh, const char *tableName, psMetadata *where, psS16 code); + +psExit pxerrorGetExitStatus (); + +void pxUsage(FILE *stream, int argc, char **argv, const char *modeName, psMetadata *argSet); +bool pxGetOptions(FILE *stream, int argc, char **argv, pxConfig *config, psMetadata *modes, psMetadata *argSets); + +#define PXOPT_ADD_MODE(option, comment, modeval, argset) \ +{ \ + if (!psMetadataAddMetadata(argSets, PS_LIST_TAIL, option, 0, comment, argset)) {;\ + psError(PS_ERR_UNKNOWN, false, "failed to add argset for %s", option); \ + } \ + psFree(argset); \ +\ + if (!psMetadataAddU32(modes, PS_LIST_TAIL, option, 0, comment, modeval)) {;\ + psError(PS_ERR_UNKNOWN, false, "failed to add argset for %s", option); \ + } \ +} + +#define PXOPT_LOOKUP_STR(var, md, key, required, ret) \ +psString var; \ +{ \ + bool status; \ + \ + var = psMetadataLookupStr(&status, md, key); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", key); \ + return ret; \ + } \ + \ + if (required && (!var)) { \ + psError(PS_ERR_UNKNOWN, true, "%s is required", key); \ + return ret; \ + } \ +} + +#define PXOPT_LOOKUP_F(var, md, key, type, required, ret) \ +ps##type var; \ +{ \ + bool status; \ + \ + var = psMetadataLookup##type(&status, md, key); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", key); \ + return ret; \ + } \ + \ + if (required && isnan(var)) { \ + psError(PS_ERR_UNKNOWN, true, "%s is required", key); \ + return ret; \ + } \ +} + +#define PXOPT_LOOKUP_F32(var, md, key, required, ret) \ + PXOPT_LOOKUP_F(var, md, key, F64, required, ret) + +#define PXOPT_LOOKUP_F64(var, md, key, required, ret) \ + PXOPT_LOOKUP_F(var, md, key, F64, required, ret) + +#define PXOPT_LOOKUP_PRIMITIVE(var, md, key, type, suffix, max, required, ret) \ +type var; \ +{ \ + psMetadataItem *item = psMetadataLookup(md, key); \ + if (!item) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", key); \ + return ret; \ + } \ + psAssert(item->comment, "metadata item comment should be defined"); \ + if (required && (!psStrcasestr(item->comment, "(found)"))) { \ + psError(PS_ERR_UNKNOWN, true, "%s is required", key); \ + return ret; \ + } \ + \ + var = item->data.suffix; \ + \ +} + +#define PXOPT_LOOKUP_S16(var, md, key, required, ret) \ + PXOPT_LOOKUP_PRIMITIVE(var, md, key, psS16, S16, INT16_MAX, required, ret) + +#define PXOPT_LOOKUP_S32(var, md, key, required, ret) \ + PXOPT_LOOKUP_PRIMITIVE(var, md, key, psS32, S32, INT32_MAX, required, ret) + +#define PXOPT_LOOKUP_S64(var, md, key, required, ret) \ + PXOPT_LOOKUP_PRIMITIVE(var, md, key, psS64, S64, INT64_MAX, required, ret) + +#define PXOPT_LOOKUP_U16(var, md, key, required, ret) \ + PXOPT_LOOKUP_PRIMITIVE(var, md, key, psU16, U16, UINT16_MAX, required, ret) + +#define PXOPT_LOOKUP_U32(var, md, key, required, ret) \ + PXOPT_LOOKUP_PRIMITIVE(var, md, key, psU32, U32, UINT32_MAX, required, ret) + +#define PXOPT_LOOKUP_U64(var, md, key, required, ret) \ + PXOPT_LOOKUP_PRIMITIVE(var, md, key, psU64, U64, UINT64_MAX, required, ret) + +#define PXOPT_LOOKUP_TIME(var, md, key, required, ret) \ +psTime *var; \ +{ \ + bool status; \ + \ + var = psMetadataLookupTime(&status, md, key); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", key); \ + return ret; \ + } \ + \ + if (required && (!var)) { \ + psError(PS_ERR_UNKNOWN, true, "%s is required", key); \ + return ret; \ + } \ +} + +#define PXOPT_LOOKUP_BOOL(var, md, key, ret) \ +bool var; \ +{ \ + bool status; \ + \ + var = psMetadataLookupBool(&status, md, key); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", key); \ + return ret; \ + } \ +} + +// XXX the PXOPT_COPY_* macros free 'to' on error + +#define PXOPT_COPY_PRIMITIVE(from, to, type, suffix, oldname, newname, newcomment) \ +{ \ + psMetadataItem *item = psMetadataLookup(from, oldname); \ + if (!item) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", oldname); \ + return false; \ + } \ + psAssert(item->comment, "metadata item comment should be defined"); \ +\ + if (psStrcasestr(item->comment, "(found)")) { \ + if (!psMetadataAdd##suffix(to, PS_LIST_TAIL, newname, PS_META_DUPLICATE_OK, newcomment, item->data.suffix)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " newname); \ + psFree(to); \ + return false; \ + } \ + } \ +} + +#define PXOPT_COPY_V(from, to, type, suffix, oldname, newname, comment) \ +{ \ + bool status = false; \ + type var = psMetadataLookup##suffix(&status, from, oldname); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for " oldname); \ + return false; \ + } \ + if (var) { \ + if (!psMetadataAdd##suffix(to, PS_LIST_TAIL, newname, PS_META_DUPLICATE_OK, comment, var)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " newname); \ + psFree(to); \ + return false; \ + } \ + }\ +} + +#define PXOPT_COPY_F(from, to, type, oldname, newname, comment) \ +{ \ + bool status = false; \ + ps##type var = psMetadataLookup##type(&status, from, oldname); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for " oldname); \ + return false; \ + } \ + if (!isnan(var)) { \ + if (!psMetadataAdd##type(to, PS_LIST_TAIL, newname, PS_META_DUPLICATE_OK, comment, var)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " newname); \ + psFree(to); \ + return false; \ + } \ + } \ +} + +#define PXOPT_COPY_F32(from, to, oldname, newname, comment) \ + PXOPT_COPY_F(from, to, F32, oldname, newname, comment) + +#define PXOPT_COPY_F64(from, to, oldname, newname, comment) \ + PXOPT_COPY_F(from, to, F64, oldname, newname, comment) + +#define PXOPT_COPY_TIME(from, to, oldname, newname, comment) \ + PXOPT_COPY_V(from, to, psTime *, Time, oldname, newname, comment) + +#define PXOPT_COPY_STR(from, to, oldname, newname, comment) \ + PXOPT_COPY_V(from, to, psString, Str, oldname, newname, comment) + +#define PXOPT_COPY_S16(from, to, oldname, newname, comment) \ + PXOPT_COPY_PRIMITIVE(from, to, psS16, S16, oldname, newname, comment) + +#define PXOPT_COPY_S32(from, to, oldname, newname, comment) \ + PXOPT_COPY_PRIMITIVE(from, to, psS32, S32, oldname, newname, comment) + +#define PXOPT_COPY_S64(from, to, oldname, newname, comment) \ + PXOPT_COPY_PRIMITIVE(from, to, psS64, S64, oldname, newname, comment) + +#define PXOPT_COPY_U16(from, to, oldname, newname, comment) \ + PXOPT_COPY_PRIMITIVE(from, to, psU16, U16, oldname, newname, comment) + +#define PXOPT_COPY_U32(from, to, oldname, newname, comment) \ + PXOPT_COPY_PRIMITIVE(from, to, psU32, U32, oldname, newname, comment) + +#define PXOPT_COPY_U64(from, to, oldname, newname, comment) \ + PXOPT_COPY_PRIMITIVE(from, to, psU64, U64, oldname, newname, comment) + + +/*** these PXOPT_ADD_WHERE macros are used to construct the default sql elements ***/ + +#define PXOPT_ADD_WHERE_STR(name) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, #name, 0, "==", str)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_STR_ALIAS(name,realname) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, name))) { \ + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, realname, 0, "==", str)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item %s", name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_S16(name) \ +{ \ + psS16 s16 = 0; \ + bool status = false; \ + if ((s16= psMetadataLookupS16(&status, config->args, "-" #name))) { \ + if (!psMetadataAddS16(config->where, PS_LIST_TAIL, #name, 0, "==", s16)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_BOOL(name) \ +{ \ + bool value = false; \ + bool status = false; \ + if ((value = psMetadataLookupBool(&status, config->args, "-" #name))) { \ + if (!psMetadataAddBool(config->where, PS_LIST_TAIL, #name, 0, "==", value)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_S32(name) \ +{ \ + psS32 s32 = 0; \ + bool status = false; \ + if ((s32= psMetadataLookupS32(&status, config->args, "-" #name))) { \ + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, #name, 0, "==", s32)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_S64(name) \ +{ \ + psS64 s64 = 0; \ + bool status = false; \ + if ((s64= psMetadataLookupS64(&status, config->args, "-" #name))) { \ + if (!psMetadataAddS64(config->where, PS_LIST_TAIL, #name, 0, "==", s64)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_F32(name) \ +{ \ + psF32 var = 0; \ + bool status = false; \ + if ((var = psMetadataLookupF32(&status, config->args, "-" #name))) { \ + if (!isnan(var)) { \ + if (!psMetadataAddF32(config->where, PS_LIST_TAIL, #name, 0, "==", var)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_F64(name) \ +{ \ + psF64 var = 0; \ + bool status = false; \ + if ((var = psMetadataLookupF64(&status, config->args, "-" #name))) { \ + if (!isnan(var)) { \ + if (!psMetadataAddF64(config->where, PS_LIST_TAIL, #name, 0, "==", var)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_BOOL_ALIAS(flag,name) \ +{ \ + bool value = 0; \ + bool status = false; \ + if ((value = psMetadataLookupBool(&status, config->args, flag))) { \ + if (!psMetadataAddBOOL(config->where, PS_LIST_TAIL, name, 0, "==", value)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item %s", flag); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_S16_ALIAS(flag,name) \ +{ \ + psS16 s16 = 0; \ + bool status = false; \ + if ((s16= psMetadataLookupS16(&status, config->args, flag))) { \ + if (!psMetadataAddS16(config->where, PS_LIST_TAIL, name, 0, "==", s16)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item %s", flag); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_S32_ALIAS(flag,name) \ +{ \ + psS32 s32 = 0; \ + bool status = false; \ + if ((s32= psMetadataLookupS32(&status, config->args, flag))) { \ + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, name, 0, "==", s32)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item %s", flag); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_S64_ALIAS(flag,name) \ +{ \ + psS64 s64 = 0; \ + bool status = false; \ + if ((s64= psMetadataLookupS64(&status, config->args, flag))) { \ + if (!psMetadataAddS64(config->where, PS_LIST_TAIL, name, 0, "==", s64)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item %s", flag); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_F32_ALIAS(flag,name) \ +{ \ + psF32 var = 0; \ + bool status = false; \ + if ((var = psMetadataLookupF32(&status, config->args, flag))) { \ + if (!isnan(var)) { \ + if (!psMetadataAddF32(config->where, PS_LIST_TAIL, name, 0, "==", var)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item %s", flag); \ + psFree(config); \ + return NULL; \ + } \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_F64_ALIAS(flag,name) \ +{ \ + psF64 var = 0; \ + bool status = false; \ + if ((var = psMetadataLookupF64(&status, config->args, flag))) { \ + if (!isnan(var)) { \ + if (!psMetadataAddF64(config->where, PS_LIST_TAIL, name, 0, "==", var)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item %s", flag); \ + psFree(config); \ + return NULL; \ + } \ + } \ + } \ +} + +#define PXOPT_ADD_WHERE_TIME_STR(name) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + psTime *time = psTimeFromISO(str, PS_TIME_UTC); \ + if (!time) { \ + psError(PS_ERR_UNKNOWN, false, "failed to convert " #name " into a psTime object"); \ + psFree(config); \ + return NULL; \ + } \ + psMetadataItem *item = psMetadataLookup(config->args, "-" #name); \ + if (item) { \ + str = item->comment; \ + } else { \ + str = NULL; \ + } \ + if (!psMetadataAddTime(config->where, PS_LIST_TAIL, #name, 0, str, time)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + psFree(time); \ + } \ +} + +#endif // PXTOOLS_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtoolsErrorCodes.c.in =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtoolsErrorCodes.c.in (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtoolsErrorCodes.c.in (revision 22161) @@ -0,0 +1,25 @@ +/* + * The line + { PXTOOLS_ERR_$X{ErrorCode}, "$X{ErrorDescription}"}, + * (without the Xs) + * will be replaced by values from errorCodes.dat + */ +#include "pslib.h" +#include "pxtoolsErrorCodes.h" + +void pxtoolsErrorRegister(void) +{ + static psErrorDescription errors[] = { + { PXTOOLS_ERR_BASE, "First value we use; lower values belong to psLib" }, + { PXTOOLS_ERR_${ErrorCode}, "${ErrorDescription}"}, + }; + static int nerror = PXTOOLS_ERR_NERROR - PXTOOLS_ERR_BASE; // number of values in enum + + for (int i = 0; i < nerror; i++) { + psErrorDescription *tmp = psAlloc(sizeof(psErrorDescription)); + *tmp = errors[i]; + psErrorRegister(tmp, 1); + psFree(tmp); /* it's on the internal list */ + } + nerror = 0; // don't register more than once +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtoolsErrorCodes.dat =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtoolsErrorCodes.dat (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtoolsErrorCodes.dat (revision 22161) @@ -0,0 +1,10 @@ +# +# This file is used to generate pxtoolsErrorCodes.h +# +BASE = 1100 First value we use; lower values belong to psLib +UNKNOWN Unknown PM error code +ARGUMENTS Incorrect arguments +SYS System error +CONFIG Problem in configure files +PROG Programming error +DATA invalid data Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtoolsErrorCodes.h.in =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtoolsErrorCodes.h.in (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtoolsErrorCodes.h.in (revision 22161) @@ -0,0 +1,18 @@ +#if !defined(PXTOOLS_ERROR_CODES_H) +#define PXTOOLS_ERROR_CODES_H +/* + * The line + * PXTOOLS_ERR_$X{ErrorCode}, + * (without the X) + * + * will be replaced by values from errorCodes.dat + */ +typedef enum { + PXTOOLS_ERR_BASE = 512, + PXTOOLS_ERR_${ErrorCode}, + PXTOOLS_ERR_NERROR +} pxtoolsErrorCode; + +void pxtoolsErrorRegister(void); + +#endif Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtree.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtree.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtree.c (revision 22161) @@ -0,0 +1,239 @@ +/* + * pxio.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pxtree.h" + +static void pxNodeFree(pxNode *node) +{ + psFree(node->name); + psFree(node->parent); + psFree(node->children); + psFree(node->data); +} + +pxNode *pxNodeAlloc(const char *name) +{ + PS_ASSERT_PTR_NON_NULL(name, NULL); + + pxNode *node = psAlloc(sizeof(pxNode)); + + node->name = psStringCopy(name); + node->parent = NULL; + node->children = psListAlloc(NULL); + node->data = NULL; + + psMemSetDeallocator(node, (psFreeFunc) pxNodeFree); + + return node; +} + +pxNode *pxNodeAddParent(pxNode *node, pxNode *parent) +{ + // add this node to it's parent's children list + psListAdd(parent->children, PS_LIST_TAIL, node); + // and set the parent pointer + node->parent = psMemIncrRefCounter(parent); + + return node; +} + +pxNode *pxNodeAddChild(pxNode *node, pxNode *child) +{ + // add the child node to this nodes list of children + psListAdd(node->children, PS_LIST_TAIL, child); + // if the child already has a parent, release the ref count + if (child->parent) { + psFree(child->parent); + } + // [re]parent the child + child->parent = psMemIncrRefCounter(node); + + return node; +} + +pxNode *pxNodeAddData(pxNode *node, psPtr data) +{ + node->data = psMemIncrRefCounter(data); + + return node; +} + +void pxNodePrint(FILE *stream, pxNode *node) +{ + fprintf(stream, "node name: %s\n", node->name); + fprintf(stream, "node parent's name: %s\n", node->parent + ? node->parent->name + : NULL); + psListIterator *iter = psListIteratorAlloc(node->children, 0, false); + pxNode *child = NULL; + while ((child = psListGetAndIncrement(iter))) { + fprintf(stream, "node child's name: %s\n", child->name); + } + psFree(iter); +} + + +static bool printNodes(void *arg, pxNode *node) +{ + pxNodePrint((FILE *)arg, node); + fprintf((FILE *)arg, "\n\n"); + + return true; +} + + +void pxTreePrint(FILE *stream, pxNode *root) +{ + pxTreeCrawl(root, printNodes, (void *)stream); +} + +// func() returning false means decend no futher along this branch + +bool pxTreeCrawl(pxNode *node, pxNodeFunc func, void *arg) +{ + // if func() returns false, stop + if (!func(arg, node)) { + return false; + } + + psListIterator *iter = psListIteratorAlloc(node->children, 0, false); + pxNode *child = NULL; + while ((child = psListGetAndIncrement(iter))) { + pxTreeCrawl(child, func, arg); + } + psFree(iter); + + return true; +} + +bool pxNodeFuncCountDependants(void *arg, pxNode *node) +{ + (*(int *)arg)++; + return true; +} + +bool pxNodeHasChildren(pxNode *node) +{ + int children = psListLength(node->children); + + return children ? true : false; +} + +bool pxNodeHasGrandChildren(pxNode *node) +{ + // find out how many nodes there are lower in the tree + // subtract the parent node from the count so we have just a tally of + // decendants + int nodes = -1; + pxTreeCrawl(node, pxNodeFuncCountDependants, &nodes); + psTrace("pxtree", PS_LOG_INFO, "node %s has %d dependants", node->name, nodes); + + // find out how many children this node has + int children = psListLength(node->children); + psTrace("pxtree", PS_LOG_INFO, "node %s has %d children", node->name, children); + + if (!children) { + // no children + return false; + } + + if (nodes < children) { + psAbort("you can't have fewer children than decendants!"); + } + + if (nodes == children) { + // no grandchildren + return false; + } + + return true; +} + +pxNode *pxTreeFromMetadata(psMetadata *md) +{ + psHash *forest = psHashAlloc(10); + + psMetadataIterator *iter = psMetadataIteratorAlloc(md, 0, NULL); + psMetadataItem *item = NULL; + while ((item = psMetadataGetAndIncrement(iter))) { + if (!(item->type == PS_DATA_STRING)) { + continue; + } + + // add this node to the forest and hopefully it'll get attached to the + // root tree + pxTreeBuilder(forest, item->name, item->data.str, NULL); + } + psFree(iter); + + pxNode *root = psHashLookup(forest, "root"); + psFree(forest); + + return psMemIncrRefCounter(root); +} + +psHash *pxTreeBuilder(psHash *forest, + const char *nodeName, + const char *childName, + psPtr data) +{ + // try to find a node with this name + pxNode *node = psHashLookup(forest, nodeName); + if (!node) { + // create a new node with this name + node = pxNodeAlloc(nodeName); + if (data) { + pxNodeAddData(node, data); + } + // add it to the hash of nodes + psHashAdd(forest, nodeName, node); + // node may be used below because it will still have a ref count of + // 1 from being on the hash, this is equivalent to the "view" we + // get from a hashlookup + psFree(node); + } else if (!node->data) { + pxNodeAddData(node, data); + } + + // does this node declare a child? + if (childName) { + // try to find a node with this name + pxNode *child = psHashLookup(forest, childName); + if (!child ) { + // create a new node with this name + child = pxNodeAlloc(childName); + // add it to the hash of nodes + psHashAdd(forest, childName, child); + // child may be used below because it will still have a ref + // count of 1 from being on the hash, this is equivalent to the + // "view" we get from a hashlookup + psFree(child); + } + pxNodeAddChild(node, child); + } + + return forest; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtree.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtree.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxtree.h (revision 22161) @@ -0,0 +1,72 @@ +/* + * pxtree.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXTREE_H +#define PXTREE_H 1 + +#include +#include + +#include + + +typedef struct pxNode { + char *name; + struct pxNode *parent; + psList *children; + void *data; +} pxNode; + +typedef bool (*pxNodeFunc)(void *arg, pxNode *node); + +pxNode *pxNodeAlloc( + const char *name +) PS_ATTR_MALLOC; + +pxNode *pxNodeAddParent(pxNode *node, pxNode *parent); +pxNode *pxNodeAddChild(pxNode *node, pxNode *child); +pxNode *pxNodeAddData(pxNode *node, psPtr data); + +void pxTreePrint(FILE *stream, pxNode *root); + +void pxNodePrint( + FILE *stream, + pxNode *node +); + +bool pxTreeCrawl( + pxNode *node, + pxNodeFunc func, + void *arg +); + +bool pxNodeFuncCountDependants(void *arg, pxNode *node); + +bool pxNodeHasChildren(pxNode *node); + +bool pxNodeHasGrandChildren(pxNode *node); + +pxNode *pxTreeFromMetadata(psMetadata *md); + +psHash *pxTreeBuilder(psHash *forest, + const char *nodeName, + const char *childName, + psPtr data); + +#endif // PXTREE_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxwarp.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxwarp.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxwarp.c (revision 22161) @@ -0,0 +1,157 @@ +/* + * pxwarp.c + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxwarp.h" + +bool pxwarpRunSetState(pxConfig *config, psS64 warp_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid warpRun state: %s", state); + return false; + } + + char *query = "UPDATE warpRun SET state = '%s' WHERE warp_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, warp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for warp_id %" PRId64, warp_id); + return false; + } + + return true; +} + + +bool pxwarpRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid chipRun state: %s", state); + return false; + } + + psString query = psStringCopy("UPDATE warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET warpRun.state = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + if (whereClause && strlen(whereClause) > 0) { + psStringAppend(&query, " %s", whereClause); + } + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxwarpRunSetLabel(pxConfig *config, psS64 warp_id, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + char *query = "UPDATE warpRun SET warpRun.label = '%s' WHERE warp_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, label, warp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change label for warp_id %" PRId64, warp_id); + return false; + } + + return true; +} + + +bool pxwarpRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + psString query = psStringCopy("UPDATE warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET warpRun.label = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxwarpQueueByFakeID(pxConfig *config, + psS64 fake_id, + char *workdir, + char *label, + char *dvodb, + char *tess_id, + char *end_stage) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // depend on the f-keys to make sure we have a valid cam_id + if (!warpRunInsert(config->dbh, + 0, // ID + fake_id, + "warp", // mode + "new", // state + workdir, + "dirty", // workdir_state + label, + dvodb, + tess_id, + end_stage, + NULL, // registered + false // magiced + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return true; + } + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pxwarp.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pxwarp.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pxwarp.h (revision 22161) @@ -0,0 +1,40 @@ +/* + * pxwarp.h + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXWARP_H +#define PXWARP_H 1 + +#include + +#include "pxtools.h" + +bool pxwarpRunSetState(pxConfig *config, psS64 warp_id, const char *state); +bool pxwarpRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); +bool pxwarpRunSetLabel(pxConfig *config, psS64 warp_id, const char *label); +bool pxwarpRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); + +bool pxwarpQueueByFakeID(pxConfig *config, + psS64 fake_id, + char *workdir, + char *label, + char *dvodb, + char *tess_id, + char *end_stage); + +#endif // PXWARP_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pzgetexp.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pzgetexp.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pzgetexp.c (revision 22161) @@ -0,0 +1,368 @@ +/* + * pzgetexp.c + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pzgetexp.h" + +#define PRODUCT_LS_CMD "dsproductls" + +static bool go (pxConfig *config); +static psArray *parseFileSets(pxConfig *config, const char *str); + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pzgetexpConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + if (!go(config)) { + goto FAIL; + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool go(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // required + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + + //optional + PXOPT_LOOKUP_S32(timeout, config->args, "-timeout", false, false); + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + // find last fileset/exp_name (if we have one) + bool haveLastFileSet = false; + psString lastFileSet = NULL; + + // -all means "request all known filesets" + if (!all) { + char *query = "SELECT * from summitExp WHERE camera = \"%s\" and TELESCOPE = \"%s\" ORDER BY dateobs DESC LIMIT 1"; + if (!p_psDBRunQuery(config->dbh, query, camera, telescope)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, false, "no summitExp rows found"); + haveLastFileSet = false; + psFree(output); + } else { + haveLastFileSet = true; + bool status = false; + lastFileSet = psStringCopy(psMetadataLookupStr(&status, output->data[0], "exp_name")); + psFree(output); + } + } + + // invoke dsproductls + // dsproductls --uri <> --last_fileset <> + psString cmd = NULL; + if (haveLastFileSet) { + psStringAppend(&cmd, "%s --uri %s --last_fileset %s", + PRODUCT_LS_CMD, uri, lastFileSet); + psFree(lastFileSet); + } else { + psStringAppend(&cmd, "%s --uri %s", PRODUCT_LS_CMD, uri); + } + if (timeout) { + psStringAppend(&cmd, " --timeout %d", timeout); + } + + psTrace("pzgetexp", PS_LOG_INFO, "cmd is: %s\n", cmd); + + FILE *output = popen(cmd, "r"); + psFree(cmd); + + if (!output) { + psError(PS_ERR_UNKNOWN, true, "popen() failed"); + return false; + } + psString cmdOutput = psSlurpFile(output); + pclose(output); + + psArray *newSummitExps = parseFileSets(config, cmdOutput); + psFree(cmdOutput); + if (!newSummitExps) { + // XXX not nessicarily an error + psError(PS_ERR_UNKNOWN, true, "no new fileSet/exp IDs"); + return false; + } + if (!psArrayLength(newSummitExps)) { + psTrace("pzgetexp", PS_LOG_INFO, "no new fileSet/exp IDs"); + psFree(newSummitExps); + return true; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newSummitExps); + return false; + } + + // increase memory table size limits; deafult is 16MB + { + // 512MB + char *query = "SET max_heap_table_size = 1024*1024*512"; + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newSummitExps); + return false; + } + } + + // create a temporry table + { + char *query = "CREATE TEMPORARY TABLE incoming" + " (exp_name VARCHAR(64), camera VARCHAR(64), telescope VARCHAR(64), dateobs DATETIME, exp_type VARCHAR(64), uri VARCHAR(255), PRIMARY KEY(exp_name, camera, telescope))" + " ENGINE=MEMORY"; + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newSummitExps); + return false; + } + } + + { + char *query = "INSERT INTO incoming (exp_name, camera, telescope, dateobs, exp_type, uri) VALUES (?, ?, ?, ?, ?, ?)"; + + long inserted = p_psDBRunQueryPrepared(config->dbh, newSummitExps, query); + if (inserted < 0) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newSummitExps); + return false; + } + // sanity check that we actually inserted something + if (inserted == 0) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error -- we should have inserted at least one row"); + psFree(newSummitExps); + return false; + } + } + + psFree(newSummitExps); + + // add new exps to summitExp + { + char *query = + "INSERT INTO summitExp" + " SElECT" + " incoming.*," + " NULL," // imfiles + " 0," // fault + " NULL" // epoch + " FROM incoming" + " LEFT JOIN summitExp" + " USING(exp_name, camera, telescope)" + " WHERE" + " summitExp.exp_name is NULL" + " AND summitExp.camera is NULL" + " AND summitExp.telescope is NULL"; + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static psArray *parseFileSets(pxConfig *config, const char *str) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(str, NULL); + + // these are constants for all records parsed -- look them up before we do + // any work + // required + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); + + // split the string into lines + psList *doc = psStringSplit(str, "\n", false); + + psListIterator *lineCursor = psListIteratorAlloc(doc, 0, false); + + psArray *summitExps = psArrayAllocEmpty(psListLength(doc)); + psString line; + while ((line = psListGetAndIncrement(lineCursor))) { + psTrace("pzgetimfile", PS_LOG_INFO, "parsing line: %s\n", line); + + // split line into tokens + psList *tokens = psStringSplit(line, " ", false); + + // check to see if this line is a comment (or if the first token is + // NULL) + if (!psListGet(tokens, 0) || *((char *)psListGet(tokens, 0)) == '#') { + psFree(tokens); + continue; + } + + // check that we have the right number of tokens + // print "# uri fileset datetime type\n"; + if (psListLength(tokens) != 4) { + // error + return false; + } + + // find the values of interest + psListIterator *tokenCursor = psListIteratorAlloc(tokens, 0, false); + char *uri = psListGetAndIncrement(tokenCursor); + char *exp_name = psListGetAndIncrement(tokenCursor); // fileset + char *dateobsStr= psListGetAndIncrement(tokenCursor); // datetime + char *exp_type = psListGetAndIncrement(tokenCursor); // type + + // create a new metadata to represent this line and it's values + psMetadata *md = psMetadataAlloc(); + if (!psMetadataAddStr(md, PS_LIST_TAIL, "exp_name", 0, NULL, exp_name)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_name"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "camera", 0, NULL, camera)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "telescope", 0, NULL, telescope)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item telescope"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + psTime *dateobs = psTimeFromISO(dateobsStr, PS_TIME_UTC); + if (!psMetadataAddTime(md, PS_LIST_TAIL, "dateobs", 0, NULL, dateobs)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item telescope"); + psFree(dateobs); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + psFree(dateobs); + if (!psMetadataAddStr(md, PS_LIST_TAIL, "exp_type", 0, NULL, exp_type)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_type"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "uri", 0, NULL, uri)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item uri"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + + // must be freed after the new metadata is built -- holds the strings + psFree(tokenCursor); + psFree(tokens); + + // add the new metadata to the result set + psArrayAdd(summitExps, 0, md); + + // debugging + if (psTraceGetLevel("pzgetexp") == PS_LOG_INFO) { + psString doc = psMetadataConfigFormat(md); + psTrace("pzgetexp", PS_LOG_INFO, "parsed line as:\n %s\n", doc); + psFree(doc); + } + + psFree(md); + + } + + psFree(lineCursor); + psFree(doc); + + return summitExps; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pzgetexp.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pzgetexp.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pzgetexp.h (revision 22161) @@ -0,0 +1,27 @@ +/* + * pzgetexp.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PZGETEXP_H +#define PZGETEXP_H 1 + +#include "pxtools.h" + +pxConfig *pzgetexpConfig(pxConfig *config, int argc, char **argv); + +#endif // PZGETEXP_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pzgetexpConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pzgetexpConfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pzgetexpConfig.c (revision 22161) @@ -0,0 +1,88 @@ +/* + * pzgetexpConfig.c + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pxtools.h" + +pxConfig *pzgetexpConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); + psFree(config); + return NULL; + } + + psMetadata *args = psMetadataAlloc(); + psMetadataAddStr(args , PS_LIST_TAIL, "-uri", 0, + "DataStore product URI (required)", ""); + psMetadataAddStr(args , PS_LIST_TAIL, "-inst", 0, + "camera name (required)", ""); + psMetadataAddStr(args , PS_LIST_TAIL, "-telescope", 0, + "telescope name (required)", ""); + psMetadataAddS32(args, PS_LIST_TAIL, "-timeout", 0, + "HTTP timeout", 0); + psMetadataAddBool(args, PS_LIST_TAIL, "-all", 0, + "download ALL filesets", 0); + + + bool status = false; + if (!psArgumentParse(args, &argc, argv) + || argc != 1 + || strcmp(psMetadataLookupStr(&status, args, "-uri"), "") == 0 + || strcmp(psMetadataLookupStr(&status, args, "-inst"), "") == 0 + || strcmp(psMetadataLookupStr(&status, args, "-telescope"), "") == 0 + ) { + fprintf(stderr, "error parsing arguments\n"); + printf("\nPan-STARRS Phase Z Get Exposures Tool\n"); + printf("Usage: %s -uri -inst -telescope [-timeout ]\n\n", + argv[0]); + psArgumentHelp(args); + psFree(config); + return NULL; + } + + config->args = args; + // don't free args here as it's silly to increment the ref count then + // "free" it + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if(!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't connect to db\n"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pzgetimfiles.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pzgetimfiles.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pzgetimfiles.c (revision 22161) @@ -0,0 +1,482 @@ +/* + * pzgetimfiles.c + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtag.h" +#include "pxtools.h" +#include "pzgetimfiles.h" + +#define FILESET_LS_CMD "dsfilesetls" + +static bool go (pxConfig *config); +static psArray *parseFiles(pxConfig *config, const char *str); + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pzgetimfilesConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + if (!go(config)) { + goto FAIL; + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool go(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // required + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + PXOPT_LOOKUP_STR(filesetid, config->args, "-filesetid", true, false); + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); + + // optional + PXOPT_LOOKUP_S32(timeout, config->args, "-timeout", false, false); + + // invoke dsfilesetls + psString cmd = NULL; + psStringAppend(&cmd, "%s --uri %s", FILESET_LS_CMD, uri); + if (timeout) { + psStringAppend(&cmd, " --timeout %d", timeout); + } + + psTrace("pzgetimfiles", PS_LOG_INFO, "cmd is: %s\n", cmd); + + FILE *output = popen(cmd, "r"); + psFree(cmd); + + if (!output) { + psError(PS_ERR_UNKNOWN, true, "popen() failed"); + return false; + } + + psString cmdOutput = psSlurpFile(output); + int status = pclose(output); + + if (status != 0) { + // mark the summitExp row as faulted + if (!p_psDBRunQuery(config->dbh, "UPDATE summitExp SET fault = %d WHERE exp_name = '%s' AND camera = '%s' AND telescope = '%s'", WEXITSTATUS(status), filesetid, camera, telescope)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + + psError(PS_ERR_UNKNOWN, true, "%s failed with exit status %d", + FILESET_LS_CMD, WEXITSTATUS(status)); + psFree(cmdOutput); + return false; + } + + // prase output of dsfilesetls + psArray *newImfiles = parseFiles(config, cmdOutput); + if (!newImfiles) { + // XXX not nessicarily an error + psError(PS_ERR_UNKNOWN, true, "no new files/imfiles"); + psFree(cmdOutput); + return false; + } + psFree(cmdOutput); + + // save the number of new Imfiles; + long imfiles = psArrayLength(newImfiles); + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // if the fileset was empty (no files) then we can bail out early + if (imfiles == 0) { + psFree(newImfiles); + + char *query = + "UPDATE summitExp" + " SET imfiles = %d" + " WHERE exp_name = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, imfiles, filesetid, camera, telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // remove the pzDownloadExp entry for this exp (fileset) + { + char *query = + "DELETE FROM pzDownloadExp" + " WHERE" + " exp_name = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, filesetid, camera, telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // sanity check: we should have removed only one row + psU64 affected = psDBAffectedRows(config->dbh); + if (psDBAffectedRows(config->dbh) != 1) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row but %" PRIu64 " rows were modified", affected); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; + } + + // create a temp table + { + char *query = + "CREATE TEMPORARY TABLE incoming (exp_name VARCHAR(64), camera VARCHAR(64), telescope VARCHAR(64), file_id VARCHAR(64), bytes INT, md5sum VARCHAR(32), class VARCHAR(64), class_id VARCHAR(64), uri VARCHAR(255), PRIMARY KEY(exp_name, camera, telescope, class, class_id)) ENGINE=MEMORY"; + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newImfiles); + return false; + } + } + + // load the imfiles (files) into the temp table + { + char *query = "INSERT INTO incoming (exp_name, camera, telescope, file_id, bytes, md5sum, class, class_id, uri) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"; + + long inserted = p_psDBRunQueryPrepared(config->dbh, newImfiles, query); + if (inserted < 0) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newImfiles); + return false; + } + // sanity check that we actually inserted something + if (inserted == 0) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error -- we should have inserted at least one row"); + psFree(newImfiles); + return false; + } + } + + psFree(newImfiles); + + // copy imfiles (files) from the temp table into summitImfiles + { + char *query = + "INSERT IGNORE INTO summitImfile" + " SELECT" + " incoming.exp_name," + " incoming.camera," + " incoming.telescope," + " incoming.file_id," + " incoming.bytes," + " incoming.md5sum," + " incoming.class," + " incoming.class_id," + " incoming.uri," + " NULL" // epoch + " FROM incoming"; + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // update summitExp.imfiles (should have been NULL) + { + char *query = + "UPDATE summitExp" + " SET imfiles = (SELECT COUNT(*) FROM summitImfile" + " WHERE" + " exp_name = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'" + ")" + " WHERE" + " exp_name = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'" + " AND imfiles IS NULL"; + if (!p_psDBRunQuery(config->dbh, query, filesetid, camera, telescope, filesetid, camera, telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // sanity check: we should have updated only one row + psU64 affected = psDBAffectedRows(config->dbh); + if (psDBAffectedRows(config->dbh) != 1) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row but %" PRIu64 " rows were modified", affected); + return false; + } + } + + // add new exps to pzDownloadExp -- must be done before the new exps are + // added to summitExp because of the SQL logic + { + char *query = + "INSERT IGNORE INTO pzDownloadExp" + " SELECT" + " incoming.exp_name," + " incoming.camera," + " incoming.telescope," + " \"run\"," // state + " NULL" + " FROM incoming" + " GROUP BY" + " incoming.exp_name," + " incoming.camera," + " incoming.telescope"; + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static psArray *parseFiles(pxConfig *config, const char *str) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(str, NULL); + + // these are constants for all records parsed -- look them up before we do + // any work + PXOPT_LOOKUP_STR(exp_name, config->args, "-filesetid", true, false); + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); + + // split the string into lines + psList *doc = psStringSplit(str, "\n", false); + + psListIterator *lineCursor = psListIteratorAlloc(doc, 0, false); + + psArray *pzPendingImfiles = psArrayAllocEmpty(psListLength(doc)); + psString line; + while ((line = psListGetAndIncrement(lineCursor))) { + psTrace("pzgetimfiles", PS_LOG_INFO, "parsing line: %s\n", line); + + // split line into tokens + psList *tokens = psStringSplit(line, " ", false); + + // check to see if this line is a comment (or if the first token is + // NULL) + if (!psListGet(tokens, 0) || *((char *)psListGet(tokens, 0)) == '#') { + psFree(tokens); + continue; + } + + // check that we have the right number of tokens + // print "# uri fileid bytes md5sum type \n"; + if (!psListLength(tokens) > 5) { + psError(PS_ERR_UNKNOWN, true, "invalid line format: %s", line); + psFree(tokens); + psFree(pzPendingImfiles); + psFree(lineCursor); + psFree(doc); + return false; + } + + // find the values of interest + psListIterator *tokenCursor = psListIteratorAlloc(tokens, 0, false); + char *uri = psListGetAndIncrement(tokenCursor); + char *file_id = psListGetAndIncrement(tokenCursor); // fileid + char *bytes = psListGetAndIncrement(tokenCursor); // bytes + char *md5sum = psListGetAndIncrement(tokenCursor); // md5sum + char *class = psListGetAndIncrement(tokenCursor); // type + char *class_id = psListGetAndIncrement(tokenCursor); // chipname + + // create a new metadata to represent this line and it's values + psMetadata *md = psMetadataAlloc(); + if (!psMetadataAddStr(md, PS_LIST_TAIL, "exp_name", 0, NULL, exp_name)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_name"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "camera", 0, NULL, camera)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "telescope", 0, NULL, telescope)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item telescope"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "file_id", 0, NULL, file_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item bytes"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddS64(md, PS_LIST_TAIL, "bytes", 0, NULL, (psS64)atoll(bytes))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item bytes"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "md5sum", 0, NULL, md5sum)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item md5sum"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "class", 0, NULL, class)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "class_id", 0, NULL, class_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class_id"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "uri", 0, NULL, uri)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item uri"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + + // must be freed after the new metadata is built -- holds the strings + psFree(tokenCursor); + psFree(tokens); + + // debugging + if (psTraceGetLevel("pzgetimfiles") >= PS_LOG_INFO) { + psString doc = psMetadataConfigFormat(md); + psTrace("pzgetimfiles", PS_LOG_INFO, "parsed line as:\n %s\n", doc); + psFree(doc); + } + + psArrayAdd(pzPendingImfiles, 0, md); + + psFree(md); + } + + psFree(lineCursor); + psFree(doc); + + if (!psArrayLength(pzPendingImfiles)) { + psError(PS_ERR_UNKNOWN, false, "string contained no valid rows"); + psFree(pzPendingImfiles); + return false; + } + + return pzPendingImfiles; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pzgetimfiles.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pzgetimfiles.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pzgetimfiles.h (revision 22161) @@ -0,0 +1,27 @@ +/* + * pzgetimfiles.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PZGETIMFILES_H +#define PZGETIMFILES 1 + +#include "pxtools.h" + +pxConfig *pzgetimfilesConfig(pxConfig *config, int argc, char **argv); + +#endif // PZGETIMFILES_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pzgetimfilesConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pzgetimfilesConfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pzgetimfilesConfig.c (revision 22161) @@ -0,0 +1,89 @@ +/* + * pzgetimfilesConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pxtools.h" + +pxConfig *pzgetimfilesConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); + psFree(config); + return NULL; + } + + psMetadata *args = psMetadataAlloc(); + psMetadataAddStr(args , PS_LIST_TAIL, "-uri", 0, + "DataStore FileSet URI (required)", NULL); + psMetadataAddStr(args , PS_LIST_TAIL, "-filesetid", 0, + "FileSet ID (required)", NULL); + psMetadataAddStr(args, PS_LIST_TAIL, "-inst", 0, + "Camera ID (required)", NULL); + psMetadataAddStr(args, PS_LIST_TAIL, "-telescope", 0, + "Telescope ID (required)", NULL); + psMetadataAddS32(args, PS_LIST_TAIL, "-timeout", 0, + "HTTP timeout", 0); + + + bool status = false; + if (!psArgumentParse(args, &argc, argv) + || argc != 1 + || (psMetadataLookupStr(&status, args, "-uri") == NULL) + || (psMetadataLookupStr(&status, args, "-filesetid") == NULL) + || (psMetadataLookupStr(&status, args, "-inst") == NULL) + || (psMetadataLookupStr(&status, args, "-telescope") == NULL) + ) { + fprintf(stderr, "error parsing arguments\n"); + printf("\nPan-STARRS Phase Z Get Imfiles Tool\n"); + printf("Usage: %s -uri -filesetid -inst -telescope ]\n\n", argv[0]); + psArgumentHelp(args); + psFree(args); + psFree(config); + return NULL; + } + + config->args = args; + // don't free args here as it's silly to increment the ref count then + // "free" it + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if(!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't connect to db\n"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pztool.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pztool.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pztool.c (revision 22161) @@ -0,0 +1,916 @@ +/* + * pztool.c + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pslib.h" +#include "pxtools.h" +#include "pxdata.h" +#include "pztool.h" + +static bool adddatastoreMode(pxConfig *config); +static bool datastoreMode(pxConfig *config); +static bool seenMode(pxConfig *config); +static bool pendingExpMode(pxConfig *config); +static bool pendingImfileMode(pxConfig *config); + +static bool copydoneMode(pxConfig *config); +static bool copiedMode(pxConfig *config); +static bool updatecopiedMode(pxConfig *config); +static bool revertcopiedMode(pxConfig *config); + +static bool clearcommonfaultsMode(pxConfig *config); +static bool advanceMode(pxConfig *config); + +static bool copydoneCompleteExp(pxConfig *config, const char *exp_name, const char *camera, const char *telescope); +static psArray *pzGetPendingCameras(pxConfig *config); +static psArray *pzArrayZip(psArray *arraySet, psS64 limit); +static bool pzDownloadExpSetState(pxConfig *config, const char *exp_name, const char *camera, const char *telescope, const char *state); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pztoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(PZTOOL_MODE_ADDDATASTORE, adddatastoreMode); + MODECASE(PZTOOL_MODE_DATASTORE, datastoreMode); + MODECASE(PZTOOL_MODE_SEEN, seenMode); + MODECASE(PZTOOL_MODE_PENDINGEXP, pendingExpMode); + MODECASE(PZTOOL_MODE_PENDINGIMFILE, pendingImfileMode); + MODECASE(PZTOOL_MODE_COPYDONE, copydoneMode); + MODECASE(PZTOOL_MODE_COPIED, copiedMode); + MODECASE(PZTOOL_MODE_UPDATECOPIED, updatecopiedMode); + MODECASE(PZTOOL_MODE_REVERTCOPIED, revertcopiedMode); + MODECASE(PZTOOL_MODE_CLEARCOMMONFAULTS, clearcommonfaultsMode); + MODECASE(PZTOOL_MODE_ADVANCE, advanceMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool adddatastoreMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + + if (!pzDataStoreInsert(config->dbh, + camera, + telescope, + uri, + NULL // epoch + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool datastoreMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (!p_psDBRunQuery(config->dbh, "SELECT * FROM pzDataStore")) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pztool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pzDataStore", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool seenMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM summitExp"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "summitExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pztool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "summitExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool pendingExpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + + PXOPT_LOOKUP_BOOL(desc, config->args, "-desc", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // XXX leave this query here ? + psString query = psStringCopy( + "SELECT" + " summitExp.*" + " FROM summitExp" + " LEFT JOIN pzDownloadExp" + " USING(exp_name, camera, telescope)" + " WHERE" + " pzDownloadExp.exp_name IS NULL" + " AND pzDownloadExp.camera IS NULL" + " AND pzDownloadExp.telescope IS NULL" + " AND summitExp.fault = 0" + ); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pzDownloadExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + psStringAppend(&query, " ORDER BY summitExp.dateobs"); + if (desc) { + psStringAppend(&query, " DESC"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pztool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pzDownloadExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool pendingImfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + + PXOPT_LOOKUP_BOOL(desc, config->args, "-desc", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psArray *cameras = pzGetPendingCameras(config); + if (!cameras) { + psError(PXTOOLS_ERR_DATA, false, "failed to find any cameras"); + return false; + } + + // array to hold the aggregate results + psArray *cameraImfiles = psArrayAlloc(0); + + for (long i = 0; i < psArrayLength(cameras); i++) { + psString query = pxDataGet("pztool_pendingimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(cameraImfiles); + return false; + } + + bool status; + psString camera = psMetadataLookupStr(&status, cameras->data[i], "camera"); + psStringAppend(&query, " WHERE camera = \"%s\"", camera); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pzDownloadImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psStringAppend(&query, " ORDER BY dateobs"); + if (desc) { + psStringAppend(&query, " DESC"); + } + + // request the full "limit" from each known camera and throw away any + // "extra" rows that we may have after merging the results. This is + // a lot simplier than a complicated scheme (tried that) to attempt to + // request on the right number of rows for each camera + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(cameraImfiles); + return false; + } + psFree(query); + + psArray *result = p_psDBFetchResult(config->dbh); + if (!result) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(cameraImfiles); + return false; + } + if (!psArrayLength(result)) { + psTrace("pztool", PS_LOG_INFO, "no rows found"); + psFree(result); + continue; + } + + // add this query into the array of result set + psArrayAdd(cameraImfiles, 0, result); + psFree(result); + } + psFree(where); + + // stitch the arrays of imfiles together + psArray *output = pzArrayZip(cameraImfiles, limit); + psFree(cameraImfiles); + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pzDownloadImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool copydoneMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false); + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); + PXOPT_LOOKUP_STR(class, config->args, "-class", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + // NOTE : the rest of the command-line args are parsed in copydoneCompleteExp + + if (!pzDownloadImfileInsert(config->dbh, + exp_name, + camera, + telescope, + class, + class_id, + uri, + code, + NULL // epoch + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!copydoneCompleteExp(config, exp_name, camera, telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "copydoneCompleteExp() failed"); + return false; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool copydoneCompleteExp(pxConfig *config, const char *exp_name, const char *camera, const char *telescope) +{ + // THIS FUNCTION MUST BE INVOKED FROM INSIDE A TRANSACTION!!! + + PS_ASSERT_PTR_NON_NULL(config, false); + + // XXX this is an ugly hack! + // we are passing exp level info to a imfile level mode (-copydone) + // these options are thrown away unless we just -copydone'd the last imfile + // in an exp. + + // optional + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + + // find all exposures that have had all of their imfiles downloaded but do + // not appear in newExp + psString query = pxDataGet("pztool_find_completed_exp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XXX this is a bit ugly : could just use the PXOPT_COPY_* methods + // and not pass exp_name, camera, or telescope as additional args + psMetadata *where = psMetadataAlloc(); + if (exp_name) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_name", 0, "==", exp_name)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_name"); + psFree(where); + return false; + } + } + + if (camera) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "camera", 0, "==", camera)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); + psFree(where); + return false; + } + } + + if (telescope) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "telescope", 0, "==", telescope)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item telescope"); + psFree(where); + return false; + } + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pztool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + pzDownloadExpRow *doneExp = pzDownloadExpObjectFromMetadata(row); + if (!doneExp) { + psError(PS_ERR_UNKNOWN, false, "pzDownloadExpObjectFromMetadata() failed"); + psFree(doneExp); + psFree(output); + return false; + } + + if (!newExpInsert(config->dbh, + 0x0, // exp_id + doneExp->exp_name, // tmp_exp_name + doneExp->camera, // tmp_camera + doneExp->telescope, // tmp_telescope + "run", // state + workdir, // workdir + "dirty", // workdir state + NULL, // reduction class + dvodb, // dvodb + tess_id, // tess_id + end_stage, // end_stage + label, + NULL // epoch + ) + ) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(doneExp); + psFree(output); + return false; + } + + psS64 exp_id = psDBLastInsertID(config->dbh); + + // insert newImfiles + { + char *query = + "INSERT INTO newImfile" + " SElECT" + " %" PRId64 "," // exp_id + " pzDownloadImfile.class_id," // tmp_class_id + " pzDownloadImfile.uri," // uri + " NULL" // epoch + " FROM pzDownloadImfile" + " WHERE" + " pzDownloadImfile.exp_name = '%s'" + " AND pzDownloadImfile.camera = '%s'" + " AND pzDownloadImfile.telescope = '%s'"; + + if (!p_psDBRunQuery(config->dbh, query, exp_id, doneExp->exp_name, doneExp->camera, doneExp->telescope)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(doneExp); + psFree(output); + return false; + } + + // sanity check: we should have inserted at least one row + psU64 affected = psDBAffectedRows(config->dbh); + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected at least 1 row but %" PRIu64 " rows were modified", affected); + psFree(doneExp); + psFree(output); + return false; + } + } + + // set pzDownloadExp.state to 'stop' + if (!pzDownloadExpSetState(config, doneExp->exp_name, doneExp->camera, doneExp->telescope, "stop")) { + psError(PS_ERR_UNKNOWN, false, "failed to change pzDownloadExp.state for %s:%s:%s", doneExp->exp_name, doneExp->camera, doneExp->telescope); + psFree(doneExp); + psFree(output); + return false; + } + + psFree(doneExp); + } + + psFree(output); + + return true; +} + +static psArray *pzGetPendingCameras(pxConfig *config) +{ + // get a list of cameras we've seen exps for + if (!p_psDBRunQuery(config->dbh, "SELECT DISTINCT camera FROM pzDownloadExp")) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *cameras = p_psDBFetchResult(config->dbh); + if (!cameras) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return NULL; + } + if (!psArrayLength(cameras)) { + psTrace("pztool", PS_LOG_INFO, "no rows found"); + psFree(cameras); + return psArrayAlloc(0); + } + + return cameras; +} + +static psArray *pzArrayZip(psArray *arraySet, psS64 limit) +{ + // figure out the combined size of all arrays in the set + long setSize = 0; + for (long i = 0; i < psArrayLength(arraySet); i++) { + setSize += psArrayLength(arraySet->data[i]); + } + + // treat 0 as "no limit" + if (limit == 0) { + limit = setSize; + } + + psArray *output = psArrayAllocEmpty(limit); + // loop over each array in the set forever + for ( + // init + long counter = 0, // the total number of elements zipped so far + i = 0, // which array in the set + index = 0; // the depth into each array + // test + (counter < setSize) + && (counter < limit) + && (i < psArrayLength(arraySet)); + // incr + counter++, ++i, + i = i % psArrayLength(arraySet), + i % psArrayLength(arraySet) ? : ++index + ) { + + psArray *array = arraySet->data[i]; + // make sure that this array has not run out of elements + if (!(index < psArrayLength(array))) { + continue; + } + + psArrayAdd(output, 0, array->data[index]); + } + + return output; +} + + +static bool copiedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "inst", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psString query = psStringCopy("SELECT * FROM pzDownloadImfile"); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "WHERE pzDownloadImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "WHERE pzDownloadImfile.fault = 0"); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pzDownloadImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pztool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipPendingImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool updatecopiedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "inst", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + PXOPT_LOOKUP_S16(code, config->args, "-code", true, false); + + if (!pxSetFaultCode(config->dbh, "pzDownloadImfile", where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree(where); + + return true; +} + + +static bool revertcopiedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "inst", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString query = pxDataGet("pztool_revertcopied.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pzDownloadImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool clearcommonfaultsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + +{ + psString query = pxDataGet("pztool_revert_downloadimfile_faults.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); +} + +{ + psString query = pxDataGet("pztool_revert_fileset_faults.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); +} + + return true; +} + +static bool advanceMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // NOTE : the command-line args are parsed in copydoneCompleteExp + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!copydoneCompleteExp(config, NULL, NULL, NULL)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "copydoneCompleteExp() failed"); + return false; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + return true; +} + + +static bool pzDownloadExpSetState(pxConfig *config, const char *exp_name, const char *camera, const char *telescope, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "reg", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid pzDownloadExp state: %s", state); + return false; + } + + char *query = "UPDATE pzDownloadExp SET state = '%s' WHERE exp_name = '%s' and camera = '%s' and telescope = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, state, exp_name, camera, telescope)) { + psError(PS_ERR_UNKNOWN, false, "failed to change state for %s:%s:%s", exp_name, camera, telescope); + return false; + } + + return true; +} + + +#if 0 +static psArray *pzArrayAddArray(psArray *array, psArray *input) +{ + for (long i = 0; i < psArrayLength(input); i++) { + psArrayAdd(array, psArrayLength(input), input->data[i]); + } + + return array; +} +#endif Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pztool.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pztool.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pztool.h (revision 22161) @@ -0,0 +1,42 @@ +/* + * pztool.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PZTOOL_H +#define PZTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + PZTOOL_MODE_NONE = 0x0, + PZTOOL_MODE_ADDDATASTORE, + PZTOOL_MODE_DATASTORE, + PZTOOL_MODE_SEEN, + PZTOOL_MODE_PENDINGEXP, + PZTOOL_MODE_PENDINGIMFILE, + PZTOOL_MODE_COPYDONE, + PZTOOL_MODE_COPIED, + PZTOOL_MODE_UPDATECOPIED, + PZTOOL_MODE_REVERTCOPIED, + PZTOOL_MODE_CLEARCOMMONFAULTS, + PZTOOL_MODE_ADVANCE +} pztoolMode; + +pxConfig *pztoolConfig(pxConfig *config, int argc, char **argv); + +#endif // PZTOOL_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/pztoolConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/pztoolConfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/pztoolConfig.c (revision 22161) @@ -0,0 +1,175 @@ +/* + * pztoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pztool.h" + +pxConfig *pztoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); + psFree(config); + return NULL; + } + + // -adddatastore + psMetadata *adddatastoreArgs = psMetadataAlloc(); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID", NULL); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID", NULL); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-uri", 0, "define storage uri", NULL); + + // -datastore + psMetadata *datastoreArgs = psMetadataAlloc(); + psMetadataAddBool(datastoreArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -seen + psMetadata *seenArgs = psMetadataAlloc(); + psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); + psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID", NULL); + psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID", NULL); + psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-exp_type", 0, "define exposure type", NULL); + psMetadataAddBool(seenArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-exp_type", 0, "define exposure type", NULL); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-desc", 0, "sort ouput in descending format", false); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -pendingimfile + psMetadata *pendingimfileArgs = psMetadataAlloc(); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-exp_type", 0, "define exposure type", NULL); + psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-desc", 0, "sort ouput in descending format", false); + psMetadataAddU64(pendingimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -copydone + psMetadata *copydoneArgs = psMetadataAlloc(); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID (required)", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID (required)", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID (required)", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-class", 0, "define class", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-class_id", 0, "define class_id", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-uri", 0, "define storage uri", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-workdir", 0, "define the \"default\" workdir for this exposure", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-dvodb", 0, "define the dvodb for the next processing step", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-tess_id", 0, "define the tess_id for the next processing step", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-end_stage", 0, "define the end goal processing step", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-label", 0, "define the label for the chip stage", NULL); + psMetadataAddS16(copydoneArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -copied + psMetadata *copiedArgs = psMetadataAlloc(); + psMetadataAddStr(copiedArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); + psMetadataAddStr(copiedArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID", NULL); + psMetadataAddStr(copiedArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID", NULL); + psMetadataAddStr(copiedArgs, PS_LIST_TAIL, "-class", 0, "define class", NULL); + psMetadataAddStr(copiedArgs, PS_LIST_TAIL, "-class_id", 0, "define class_id", NULL); + psMetadataAddBool(copiedArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddU64(copiedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(copiedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updatecopied + psMetadata *updatecopiedArgs = psMetadataAlloc(); + psMetadataAddStr(updatecopiedArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID (required)", NULL); + psMetadataAddStr(updatecopiedArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID (required)", NULL); + psMetadataAddStr(updatecopiedArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID (required)", NULL); + psMetadataAddStr(updatecopiedArgs, PS_LIST_TAIL, "-class", 0, "define class", NULL); + psMetadataAddStr(updatecopiedArgs, PS_LIST_TAIL, "-class_id", 0, "define class_id", NULL); + psMetadataAddS16(updatecopiedArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -revertcopied + psMetadata *revertcopiedArgs = psMetadataAlloc(); + psMetadataAddStr(revertcopiedArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID (required)", NULL); + psMetadataAddStr(revertcopiedArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID (required)", NULL); + psMetadataAddStr(revertcopiedArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID (required)", NULL); + psMetadataAddStr(revertcopiedArgs, PS_LIST_TAIL, "-class", 0, "define class", NULL); + psMetadataAddStr(revertcopiedArgs, PS_LIST_TAIL, "-class_id", 0, "define class_id", NULL); + psMetadataAddS16(revertcopiedArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -clearcommonfaults + psMetadata *clearcommonfaultsArgs = psMetadataAlloc(); + + // -advance + psMetadata *advanceArgs = psMetadataAlloc(); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-workdir", 0, "define the \"default\" workdir for this exposure", NULL); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-dvodb", 0, "define the dvodb for the next processing step", NULL); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-tess_id", 0, "define the tess_id for the next processing step", NULL); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-end_stage", 0, "define the end goal processing step", NULL); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-label", 0, "define the label for the chip stage", NULL); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-adddatastore", "", PZTOOL_MODE_ADDDATASTORE, adddatastoreArgs); + PXOPT_ADD_MODE("-datastore", "", PZTOOL_MODE_DATASTORE, datastoreArgs); + PXOPT_ADD_MODE("-seen", "", PZTOOL_MODE_SEEN, seenArgs); + PXOPT_ADD_MODE("-pendingexp", "", PZTOOL_MODE_PENDINGEXP, pendingexpArgs); + PXOPT_ADD_MODE("-pendingimfile", "", PZTOOL_MODE_PENDINGIMFILE,pendingimfileArgs); + PXOPT_ADD_MODE("-copydone", "", PZTOOL_MODE_COPYDONE, copydoneArgs); + PXOPT_ADD_MODE("-copied", "", PZTOOL_MODE_COPIED, copiedArgs); + PXOPT_ADD_MODE("-updatecopied", "", PZTOOL_MODE_UPDATECOPIED,updatecopiedArgs); + PXOPT_ADD_MODE("-revertcopied", "", PZTOOL_MODE_REVERTCOPIED,revertcopiedArgs); + PXOPT_ADD_MODE("-clearcommonfaults","", PZTOOL_MODE_CLEARCOMMONFAULTS,clearcommonfaultsArgs); + PXOPT_ADD_MODE("-advance", "", PZTOOL_MODE_ADVANCE, advanceArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/regtool.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/regtool.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/regtool.c (revision 22161) @@ -0,0 +1,1078 @@ +/* + * regtool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxdata.h" +#include "regtool.h" +#include "chiptool.h" + +// imfile +static bool pendingimfileMode(pxConfig *config); +static bool addprocessedimfileMode(pxConfig *config); +static bool processedimfileMode(pxConfig *config); +static bool revertprocessedimfileMode(pxConfig *config); +static bool updateprocessedimfileMode(pxConfig *config); +// exp +static bool pendingexpMode(pxConfig *config); +static bool addprocessedexpMode(pxConfig *config); +static bool processedexpMode(pxConfig *config); +static bool revertprocessedexpMode(pxConfig *config); +static bool updateprocessedexpMode(pxConfig *config); +static bool cleardupexpMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = regtoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + // imfile + MODECASE(REGTOOL_MODE_PENDINGIMFILE, pendingimfileMode); + MODECASE(REGTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileMode); + MODECASE(REGTOOL_MODE_PROCESSEDIMFILE, processedimfileMode); + MODECASE(REGTOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileMode); + MODECASE(REGTOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileMode); + // exp + MODECASE(REGTOOL_MODE_PENDINGEXP, pendingexpMode); + MODECASE(REGTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpMode); + MODECASE(REGTOOL_MODE_PROCESSEDEXP, processedexpMode); + MODECASE(REGTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); + MODECASE(REGTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + MODECASE(REGTOOL_MODE_CLEARDUPEXP, cleardupexpMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool pendingimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // select newImfiles that: + // exp_id is in newExp + // don't have their exp_id in rawExp + // XXX having the same exp_id in newExp and raw*Exp is probably an error + // that should be checked for + + psString query = pxDataGet("regtool_pendingimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms) + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "regPendingImfile", !simple)) { + psError(PXTOOLS_ERR_PROG, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false); + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); + PXOPT_LOOKUP_STR(tmp_class_id, config->args, "-tmp_class_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + + // optional + PXOPT_LOOKUP_STR(exp_type, config->args, "-exp_type", false, false); + PXOPT_LOOKUP_STR(filelevel, config->args, "-filelevel", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-filter", false, false); + PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); + + PXOPT_LOOKUP_F32(airmass, config->args, "-airmass", false, false); + PXOPT_LOOKUP_F64(ra, config->args, "-ra", false, false); + PXOPT_LOOKUP_F64(decl, config->args, "-decl", false, false); + PXOPT_LOOKUP_F32(exp_time, config->args, "-exp_time", false, false); + PXOPT_LOOKUP_F32(sat_pixel_frac, config->args, "-sat_pixel_frac", false, false); + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(alt, config->args, "-alt", false, false); + PXOPT_LOOKUP_F64(az, config->args, "-az", false, false); + PXOPT_LOOKUP_F32(ccd_temp, config->args, "-ccd_temp", false, false); + PXOPT_LOOKUP_F64(posang, config->args, "-posang", false, false); + PXOPT_LOOKUP_F32(m1_x, config->args, "-m1_x", false, false); + PXOPT_LOOKUP_F32(m1_y, config->args, "-m1_y", false, false); + PXOPT_LOOKUP_F32(m1_z, config->args, "-m1_z", false, false); + PXOPT_LOOKUP_F32(m1_tip, config->args, "-m1_tip", false, false); + PXOPT_LOOKUP_F32(m1_tilt, config->args, "-m1_tilt", false, false); + PXOPT_LOOKUP_F32(m2_x, config->args, "-m2_x", false, false); + PXOPT_LOOKUP_F32(m2_y, config->args, "-m2_y", false, false); + PXOPT_LOOKUP_F32(m2_z, config->args, "-m2_z", false, false); + PXOPT_LOOKUP_F32(m2_tip, config->args, "-m2_tip", false, false); + PXOPT_LOOKUP_F32(m2_tilt, config->args, "-m2_tilt", false, false); + + PXOPT_LOOKUP_F32(env_temp, config->args, "-env_temperature", false, false); + PXOPT_LOOKUP_F32(env_humid, config->args, "-env_humidity", false, false); + PXOPT_LOOKUP_F32(env_wind, config->args, "-env_wind_speed", false, false); + PXOPT_LOOKUP_F32(env_dir, config->args, "-env_wind_dir", false, false); + PXOPT_LOOKUP_F32(teltemp_m1, config->args, "-teltemp_m1", false, false); + PXOPT_LOOKUP_F32(teltemp_m1cell, config->args, "-teltemp_m1cell", false, false); + PXOPT_LOOKUP_F32(teltemp_m2, config->args, "-teltemp_m2", false, false); + PXOPT_LOOKUP_F32(teltemp_spider, config->args, "-teltemp_spider", false, false); + PXOPT_LOOKUP_F32(teltemp_truss, config->args, "-teltemp_truss", false, false); + PXOPT_LOOKUP_F32(teltemp_extra, config->args, "-teltemp_extra", false, false); + PXOPT_LOOKUP_F32(pon_time, config->args, "-pon_time", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + PXOPT_LOOKUP_STR(object, config->args, "-object", false, false); + PXOPT_LOOKUP_TIME(dateobs, config->args, "-dateobs", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (!rawImfileInsert( + config->dbh, + exp_id, + exp_name, + camera, + telescope, + dateobs, + tmp_class_id, + class_id, + uri, + exp_type, + filelevel, + filter, + comment, + airmass, + ra, + decl, + exp_time, + sat_pixel_frac, + bg, + bg_stdev, + bg_mean_stdev, + alt, + az, + ccd_temp, + posang, + m1_x, + m1_y, + m1_z, + m1_tip, + m1_tilt, + m2_x, + m2_y, + m2_z, + m2_tip, + m2_tilt, + env_temp, + env_humid, + env_wind, + env_dir, + teltemp_m1, + teltemp_m1cell, + teltemp_m2, + teltemp_spider, + teltemp_truss, + teltemp_extra, + pon_time, + user_1, + user_2, + user_3, + user_4, + user_5, + object, + hostname, + code, + NULL + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("regtool_processedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND rawImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND rawImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool revertprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tmp_class_id", "tmp_class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("regtool_revertprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool updateprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + PXOPT_LOOKUP_S16(code, config->args, "-code", true, false); + + if (!pxSetFaultCode(config->dbh, "rawImfile", where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + + return true; +} + + +static bool pendingexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // return only exps that: + // are not in rawExp + // have ALL of their imfiles in rawImfile (by count) + // and have no associated imfiles left in newImfile + psString query = pxDataGet("regtool_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + psString limitString = NULL; + if (limit) { + limitString = psDBGenerateLimitSQL(limit); + // skip past the "hook" comment + psStringPrepend(&limitString, "\n"); + } + + // 1st arg: where hook, 2nd arg: limit hook + if (!p_psDBRunQuery(config->dbh, query, "", limitString ? limitString : "")) { + // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms) + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(limitString); + psFree(query); + return false; + } + psFree(limitString); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "regPendingExp", !simple)) { + psError(PXTOOLS_ERR_PROG, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // make sure that the exp_id(s) are ready to be updated based on: + // exp_id is not in rawExp + // exp_id is not in newImfile + // that the correct count of imfiles is in rawImfile + + // required + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false); + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); + PXOPT_LOOKUP_STR(exp_tag, config->args, "-exp_tag", true, false); + PXOPT_LOOKUP_STR(filelevel, config->args, "-filelevel", true, false); + + // optional + PXOPT_LOOKUP_TIME(dateobs, config->args, "-dateobs", false, false); + PXOPT_LOOKUP_STR(exp_type, config->args, "-exp_type", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-filter", false, false); + PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); + PXOPT_LOOKUP_F32(airmass, config->args, "-airmass", false, false); + PXOPT_LOOKUP_F64(ra, config->args, "-ra", false, false); + PXOPT_LOOKUP_F64(decl, config->args, "-decl", false, false); + PXOPT_LOOKUP_F32(exp_time, config->args, "-exp_time", false, false); + PXOPT_LOOKUP_F32(sat_pixel_frac, config->args, "-sat_pixel_frac", false, false); + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + PXOPT_LOOKUP_F64(alt, config->args, "-alt", false, false); + PXOPT_LOOKUP_F64(az, config->args, "-az", false, false); + PXOPT_LOOKUP_F32(ccd_temp, config->args, "-ccd_temp", false, false); + PXOPT_LOOKUP_F64(posang, config->args, "-posang", false, false); + PXOPT_LOOKUP_F32(m1_x, config->args, "-m1_x", false, false); + PXOPT_LOOKUP_F32(m1_y, config->args, "-m1_y", false, false); + PXOPT_LOOKUP_F32(m1_z, config->args, "-m1_z", false, false); + PXOPT_LOOKUP_F32(m1_tip, config->args, "-m1_tip", false, false); + PXOPT_LOOKUP_F32(m1_tilt, config->args, "-m1_tilt", false, false); + PXOPT_LOOKUP_F32(m2_x, config->args, "-m2_x", false, false); + PXOPT_LOOKUP_F32(m2_y, config->args, "-m2_y", false, false); + PXOPT_LOOKUP_F32(m2_z, config->args, "-m2_z", false, false); + PXOPT_LOOKUP_F32(m2_tip, config->args, "-m2_tip", false, false); + PXOPT_LOOKUP_F32(m2_tilt, config->args, "-m2_tilt", false, false); + + PXOPT_LOOKUP_F32(env_temp, config->args, "-env_temperature", false, false); + PXOPT_LOOKUP_F32(env_humid, config->args, "-env_humidity", false, false); + PXOPT_LOOKUP_F32(env_wind, config->args, "-env_wind_speed", false, false); + PXOPT_LOOKUP_F32(env_dir, config->args, "-env_wind_dir", false, false); + PXOPT_LOOKUP_F32(teltemp_m1, config->args, "-teltemp_m1", false, false); + PXOPT_LOOKUP_F32(teltemp_m1cell, config->args, "-teltemp_m1cell", false, false); + PXOPT_LOOKUP_F32(teltemp_m2, config->args, "-teltemp_m2", false, false); + PXOPT_LOOKUP_F32(teltemp_spider, config->args, "-teltemp_spider", false, false); + PXOPT_LOOKUP_F32(teltemp_truss, config->args, "-teltemp_truss", false, false); + PXOPT_LOOKUP_F32(teltemp_extra, config->args, "-teltemp_extra", false, false); + PXOPT_LOOKUP_F32(pon_time, config->args, "-pon_time", false, false); + PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); + PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); + PXOPT_LOOKUP_F64(user_3, config->args, "-user_3", false, false); + PXOPT_LOOKUP_F64(user_4, config->args, "-user_4", false, false); + PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); + PXOPT_LOOKUP_STR(object, config->args, "-object", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + + // default + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + psString query = pxDataGet("regtool_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = NULL; + { + // build a query to search by exp_id + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddS64(where, PS_LIST_TAIL, "newExp.exp_id", 0, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + psFree(query); + return false; + } + + whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psFree(where); + if (whereClause) { + // skip past comment "hook" + psStringPrepend(&whereClause, "\n AND "); + } + } + + // 1st arg: where hook, 2nd arg: limit hook + if (!p_psDBRunQuery(config->dbh, query, whereClause ? whereClause : "", "")) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereClause); + psFree(query); + return false; + } + psFree(whereClause); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, false, "no pending newExp rows found"); + psFree(output); + return false; + } + // sanity check that we only got one row + if (psArrayLength(output) != 1) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "should have gotten 1 row but %lu rows were returned", psArrayLength(output)); + psFree(output); + return NULL; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // insert the exp into rawExp + psMetadata *row = output->data[0]; + // convert metadata into a newExp object + psMetadataConfigPrint(stdout, row); + newExpRow *newExp = newExpObjectFromMetadata(row); + psFree(output); + if (!newExp) { + psError(PS_ERR_UNKNOWN, false, "this should not happen"); + return false; + } + + // carry through these values + // new CLI options overwrite existing values + workdir = workdir ? workdir : newExp->workdir; + reduction = reduction ? reduction : newExp->reduction; + dvodb = dvodb ? dvodb : newExp->dvodb; + tess_id = tess_id ? tess_id : newExp->tess_id; + end_stage = end_stage ? end_stage : newExp->end_stage; + label = label ? label : newExp->label; + // don't free newExp until just before we return, or these refs will break + + if (!rawExpInsert(config->dbh, + exp_id, + exp_name, + camera, + telescope, + dateobs, + exp_tag, + exp_type, + filelevel, + workdir, + reduction, + dvodb, + tess_id, + end_stage, + filter, + comment, + airmass, + ra, + decl, + exp_time, + sat_pixel_frac, + bg, + bg_stdev, + bg_mean_stdev, + alt, + az, + ccd_temp, + posang, + m1_x, + m1_y, + m1_z, + m1_tip, + m1_tilt, + m2_x, + m2_y, + m2_z, + m2_tip, + m2_tilt, + env_temp, + env_humid, + env_wind, + env_dir, + teltemp_m1, + teltemp_m1cell, + teltemp_m2, + teltemp_spider, + teltemp_truss, + teltemp_extra, + pon_time, + user_1, + user_2, + user_3, + user_4, + user_5, + object, + 0.0, + hostname, + code, + NULL + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newExp); + return false; + } + + // set the state for the newExp to stop + if (!pxnewExpSetState(config, exp_id, "stop")) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to change newExp.state for exp_id: %"PRId64, exp_id); + psFree(newExp); + return false; + } + + // should we stop here and proceed on to the chip stage? + // NULL for end_stage means go as far as possible + if (end_stage && psStrcasestr(end_stage, "reg")) { + // then we are done here + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(newExp); + return true; + } + // else continue on... + + + // insert an entry into the chipPendingExp table + // this can only be run as the newExp's state has been set to stop + if (!pxchipQueueByExpTag(config, + exp_id, + workdir, + label, + reduction, + NULL, // expgroup + dvodb, + tess_id, + end_stage + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to queue chipPendingExp"); + psFree(newExp); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newExp); + return false; + } + + psFree(newExp); + return true; +} + + +static bool processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("regtool_processedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // XX test this out; need to make this consistent with the list in regtoolConfig.c + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_F32(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F32(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND rawExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND rawExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool revertprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("regtool_revertprocessedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(where); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + + +static bool updateprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + PXOPT_LOOKUP_S16(code, config->args, "-code", true, false); + + if (!pxSetFaultCode(config->dbh, "rawExp", where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree(where); + return false; + } + psFree(where); + + return true; +} + + +static bool cleardupexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + +// create temp table of exp_ids to be removed +{ + psString query = pxDataGet("regtool_create_dup_table.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); +} + +// populate that table +// XXX note that this query currently doesn't not correctly handle the case +// where there is more than one duplicate +{ + psString query = pxDataGet("regtool_populate_dup_table.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); +} + + if (!p_psDBRunQuery(config->dbh, "DELETE FROM rawImfile USING rawImfile, duplicate WHERE duplicate.exp_id = rawImfile.exp_id")) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, "DELETE FROM rawExp USING rawExp, duplicate WHERE duplicate.exp_id = rawExp.exp_id")) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, "DELETE FROM newImfile USING newImfile, duplicate WHERE duplicate.exp_id = newImfile.exp_id")) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, "DELETE FROM newExp USING newExp, duplicate WHERE duplicate.exp_id = newExp.exp_id")) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/regtool.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/regtool.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/regtool.h (revision 22161) @@ -0,0 +1,42 @@ +/* + * regtool.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef REGTOOL_H +#define REGTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + REGTOOL_MODE_NONE = 0x0, + REGTOOL_MODE_PENDINGIMFILE, + REGTOOL_MODE_ADDPROCESSEDIMFILE, + REGTOOL_MODE_PROCESSEDIMFILE, + REGTOOL_MODE_REVERTPROCESSEDIMFILE, + REGTOOL_MODE_UPDATEPROCESSEDIMFILE, + REGTOOL_MODE_PENDINGEXP, + REGTOOL_MODE_ADDPROCESSEDEXP, + REGTOOL_MODE_PROCESSEDEXP, + REGTOOL_MODE_REVERTPROCESSEDEXP, + REGTOOL_MODE_UPDATEPROCESSEDEXP, + REGTOOL_MODE_CLEARDUPEXP, +} regtoolMode; + +pxConfig *regtoolConfig(pxConfig *config, int argc, char **argv); + +#endif // REGTOOL_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/regtoolConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/regtoolConfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/regtoolConfig.c (revision 22161) @@ -0,0 +1,295 @@ +/* + * regtoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include + +#include "pxtools.h" +#include "regtool.h" + +#define ADD_OPT(TYPE,TARG,NAME,COMMENT,DEFAULT) psMetadataAdd##TYPE(TARG, PS_LIST_TAIL, NAME, 0, COMMENT, DEFAULT) + +pxConfig *regtoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PXTOOLS_ERR_CONFIG, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -pendingimfile + psMetadata *pendingimfileArgs = psMetadataAlloc(); + ADD_OPT(U64, pendingimfileArgs, "-limit", "limit result set to N items", 0); + ADD_OPT(Bool, pendingimfileArgs, "-simple", "use the simple output format", false); + + // -addprocessedimfile + psMetadata *addprocessedimfileArgs = psMetadataAlloc(); + ADD_OPT(S64, addprocessedimfileArgs, "-exp_id", "define exposure ID (required)", 0); + ADD_OPT(Str, addprocessedimfileArgs, "-exp_name", "define the exp_name (required)", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-inst", "define the camera name (required)", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-telescope", "define the telescope name (required)", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-tmp_class_id", "define temp. class ID (required)", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-class_id", "define class ID (required)", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-uri", "define URI (required)", NULL); + + ADD_OPT(Str, addprocessedimfileArgs, "-exp_type", "define exposure type", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-filelevel", "define filelevel", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-filter", "define filter ", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-comment", "define comment ", NULL); + + ADD_OPT(F32, addprocessedimfileArgs, "-airmass", "define airmass", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-ra", "define RA", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-decl", "define DEC", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-exp_time", "define exposure time", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-sat_pixel_frac", "define fraction of saturated pixels", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-bg", "define exposue background", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-bg_stdev", "define exposue background stdev", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-bg_mean_stdev", "define exposue background mean stdev", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-alt", "define altitute", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-az", "define azimuth", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-ccd_temp", "define ccd tempature", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-posang", "define rotator position angle", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m1_x", "define M1 X position", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m1_y", "define M1 Y position", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m1_z", "define M1 Z position", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m1_tip", "define M1 TIP position", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m1_tilt", "define M1 TILT position", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m2_x", "define M2 X position", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m2_y", "define M2 Y position", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m2_z", "define M2 Z position", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m2_tip", "define M2 TIP position", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-m2_tilt", "define M2 TILT position", NAN); + + ADD_OPT(F32, addprocessedimfileArgs, "-env_temperature","define Environmental Temperature", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-env_humidity", "define Environmental Humidity", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-env_wind_speed", "define Environmental Wind Speed", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-env_wind_dir", "define Environmental Wind Direction", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-teltemp_m1", "define Telescope Temperature : M1", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-teltemp_m1cell", "define Telescope Temperature : M1 Cell", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-teltemp_m2", "define Telescope Temperature : M2", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-teltemp_spider", "define Telescope Temperature : Spider", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-teltemp_truss", "define Telescope Temperature : Truss", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-teltemp_extra", "define Telescope Temperature : Extra", NAN); + ADD_OPT(F32, addprocessedimfileArgs, "-pon_time", "define time to last Power On", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-user_1", "define user statistic (1)", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-user_2", "define user statistic (2)", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-user_3", "define user statistic (3)", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-user_4", "define user statistic (4)", NAN); + ADD_OPT(F64, addprocessedimfileArgs, "-user_5", "define user statistic (5)", NAN); + ADD_OPT(Str, addprocessedimfileArgs, "-object", "define exposure object", NULL); + ADD_OPT(Time, addprocessedimfileArgs, "-dateobs", "define observation time", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-hostname", "define host name", NULL); + ADD_OPT(S16, addprocessedimfileArgs, "-code", "set fault code", 0); + + // -processedimfile + psMetadata *processedimfileArgs = psMetadataAlloc(); + ADD_OPT(S64, processedimfileArgs, "-exp_id", "search by exposure ID", 0); + ADD_OPT(Str, processedimfileArgs, "-exp_name", "search by exposure name", NULL); + ADD_OPT(Str, processedimfileArgs, "-class_id", "search by class ID", NULL); + ADD_OPT(U64, processedimfileArgs, "-limit", "limit result set to N items", 0); + ADD_OPT(Bool, processedimfileArgs, "-faulted", "only return imfiles with a fault status set", false); + ADD_OPT(Bool, processedimfileArgs, "-simple", "use the simple output format", false); + + // -revertprocessedimfile + psMetadata *revertprocessedimfileArgs = psMetadataAlloc(); + ADD_OPT(S64, revertprocessedimfileArgs, "-exp_id", "search by exposure ID (required)", 0); + ADD_OPT(Str, revertprocessedimfileArgs, "-tmp_class_id", "searcy by temp. class ID", NULL); + ADD_OPT(Str, revertprocessedimfileArgs, "-class_id", "search by class ID", NULL); + ADD_OPT(S16, revertprocessedimfileArgs, "-code", "search by fault code", 0); + + // -updateprocessedimfile + psMetadata *updateprocessedimfileArgs = psMetadataAlloc(); + ADD_OPT(S64, updateprocessedimfileArgs, "-exp_id", "search by exposure ID", 0); + ADD_OPT(Str, updateprocessedimfileArgs, "-class_id", "search by class ID", NULL); + ADD_OPT(S16, updateprocessedimfileArgs, "-code", "set fault code (required)", INT16_MAX); + + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addprocessedexp + psMetadata *addprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "exp_id to operate on (required)", 0); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-exp_name", 0, "define the exp_name (required)", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-inst", 0, "define the camera name (required)", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-telescope", 0, "define the telescope name (required)", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-exp_tag", 0, "define the external exposure tag name (required)", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-filelevel", 0, "define the data partitioning level of this file (required)", NULL); + + psMetadataAddTime(addprocessedexpArgs, PS_LIST_TAIL, "-dateobs", 0, "define observation time", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-exp_type", 0, "define exposure type", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-workdir", 0, "define the \"default\" workdir for this exposure", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-dvodb", 0, "define the dvodb for the next processing step", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-tess_id", 0, "define the tess_id for the next processing step", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-end_stage", 0, "define the end goal processing step", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-reduction", 0, "define the \"default\" reduction class for this exposure", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-filter", 0, "define filter ", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-comment", 0, "define comment ", NULL); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-airmass", 0, "define airmass", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-ra", 0, "define RA", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-decl", 0, "define DEC", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-exp_time", 0, "define exposure time", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-sat_pixel_frac", 0, "define fraction of saturated pixels", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg", 0, "define exposue background", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposue background stdev", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, "define exposue background mean stdev", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-alt", 0, "define altitute", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-az", 0, "define azimuth", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-ccd_temp", 0, "define ccd tempature", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-posang", 0, "define rotator position angle", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-m1_x", 0, "define M1 X position", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-m1_y", 0, "define M1 Y position", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-m1_z", 0, "define M1 Z position", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-m1_tip", 0, "define M1 TIP position", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-m1_tilt", 0, "define M1 TILT position", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-m2_x", 0, "define M2 X position", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-m2_y", 0, "define M2 Y position", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-m2_z", 0, "define M2 Z position", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-m2_tip", 0, "define M2 TIP position", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-m2_tilt", 0, "define M2 TILT position", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-env_temperature", 0, "define Environmental Temperature", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-env_humidity", 0, "define Environmental Humidity", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-env_wind_speed", 0, "define Environmental Wind Speed", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-env_wind_dir", 0, "define Environmental Wind Direction", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-teltemp_m1", 0, "define Telescope Temperature : M1", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-teltemp_m1cell", 0, "define Telescope Temperature : M1 Cell", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-teltemp_m2", 0, "define Telescope Temperature : M2", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-teltemp_spider", 0, "define Telescope Temperature : Spider", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-teltemp_truss", 0, "define Telescope Temperature : Truss", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-teltemp_extra", 0, "define Telescope Temperature : Extra", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-pon_time", 0, "define time to last Power On", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_1", 0, "define user statistic (1)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_2", 0, "define user statistic (2)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_3", 0, "define user statistic (3)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_4", 0, "define user statistic (4)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_5", 0, "define user statistic (5)", NAN); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-object", 0, "define exposure object", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-label", 0, "define label for chip stage (non-detrend data only)", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-hostname", 0, "define host name", NULL); + psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -processedexp + psMetadata *processedexpArgs = psMetadataAlloc(); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(processedexpArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(processedexpArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", "object"); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-ra_min", 0, "search by min", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-ra_max", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-decl_min", 0, "search by min", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-decl_max", 0, "search by max", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-exp_time_min", 0, "search by min", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-exp_time_max", 0, "search by max", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by max fraction of saturated pixels", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by min fraction of saturated pixels", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_min", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_max", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-alt_min", 0, "search by min", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-alt_max", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-az_min", 0, "search by min", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-az_max", 0, "search by max", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN); + psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + + psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertprocessedexp + psMetadata *revertprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID (required)", 0); + psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -updatedprocessedexp + psMetadata *updatedprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(updatedprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddS16(updatedprocessedexpArgs, PS_LIST_TAIL, "-code", 0, "set fault code (required)", INT16_MAX); + + // -cleardupexp + psMetadata *cleardupexpArgs = psMetadataAlloc(); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-pendingimfile", "", REGTOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + PXOPT_ADD_MODE("-addprocessedimfile", "", REGTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); + PXOPT_ADD_MODE("-processedimfile", "", REGTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); + PXOPT_ADD_MODE("-revertprocessedimfile", "", REGTOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileArgs); + PXOPT_ADD_MODE("-updateprocessedimfile", "", REGTOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileArgs); + PXOPT_ADD_MODE("-pendingexp", "", REGTOOL_MODE_PENDINGEXP,pendingexpArgs); + PXOPT_ADD_MODE("-addprocessedexp", "", REGTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs); + PXOPT_ADD_MODE("-processedexp", "", REGTOOL_MODE_PROCESSEDEXP, processedexpArgs); + PXOPT_ADD_MODE("-revertprocessedexp", "", REGTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpArgs); + PXOPT_ADD_MODE("-updateprocessedexp", "", REGTOOL_MODE_UPDATEPROCESSEDEXP, updatedprocessedexpArgs); + PXOPT_ADD_MODE("-cleardupexp", "", REGTOOL_MODE_CLEARDUPEXP, cleardupexpArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PXTOOLS_ERR_SYS, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/stacktool.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/stacktool.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/stacktool.c (revision 22161) @@ -0,0 +1,1127 @@ +/* + * stacktool.c + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "stacktool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool definerunMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool addinputskyfileMode(pxConfig *config); +static bool inputskyfileMode(pxConfig *config); +static bool tosumMode(pxConfig *config); +static bool addsumskyfileMode(pxConfig *config); +static bool sumskyfileMode(pxConfig *config); +static bool revertsumskyfileMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupskyfileMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); + +static bool setstackRunState(pxConfig *config, psS64 stack_id, const char *state); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = stacktoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(STACKTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(STACKTOOL_MODE_DEFINERUN, definerunMode); + MODECASE(STACKTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(STACKTOOL_MODE_ADDINPUTSKYFILE, addinputskyfileMode); + MODECASE(STACKTOOL_MODE_INPUTSKYFILE, inputskyfileMode); + MODECASE(STACKTOOL_MODE_TOSUM, tosumMode); + MODECASE(STACKTOOL_MODE_ADDSUMSKYFILE, addsumskyfileMode); + MODECASE(STACKTOOL_MODE_SUMSKYFILE, sumskyfileMode); + MODECASE(STACKTOOL_MODE_REVERTSUMSKYFILE, revertsumskyfileMode); + MODECASE(STACKTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); + MODECASE(STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode); + MODECASE(STACKTOOL_MODE_DONECLEANUP, donecleanupMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + + psMetadata *where = psMetadataAlloc(); + psMetadata *having = psMetadataAlloc(); // HAVING clause + + // select based on properties of the raw exposures + PXOPT_COPY_STR(config->args, where, "-select_inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-select_telescope", "rawExp.telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-select_filter", "rawExp.filter", "=="); + PXOPT_COPY_STR(config->args, where, "-select_uri", "rawExp.uri", "=="); + PXOPT_COPY_TIME(config->args, where, "-select_dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-select_dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_F32(config->args, where, "-select_airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F32(config->args, where, "-select_airmass_max", "rawExp.airmass", "<="); + PXOPT_COPY_F32(config->args, where, "-select_sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<="); + PXOPT_COPY_F32(config->args, where, "-select_exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-select_exp_time_max", "rawExp.exp_time", "<="); + PXOPT_COPY_F32(config->args, where, "-select_ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-select_ccd_temp_max", "rawExp.ccd_temp", "<="); + PXOPT_COPY_F32(config->args, where, "-select_posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F32(config->args, where, "-select_posang_max", "rawExp.posang", "<="); + PXOPT_COPY_F32(config->args, where, "-select_solang_min", "rawExp.solang", ">="); + PXOPT_COPY_F32(config->args, where, "-select_solang_max", "rawExp.solang", "<="); + PXOPT_COPY_STR(config->args, where, "-select_exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_F32(config->args, where, "-select_good_frac_min", "warpSkyfile.good_frac", ">="); + PXOPT_COPY_STR(config->args, where, "-select_skycell_id", "warpSkyfile.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-select_label", "warpRun.label", "=="); + + // these are used to build the HAVING restriction + PXOPT_COPY_S32(config->args, having, "-min_num", "num_warp", ">="); + + // other options applied outside of the WHERE + PXOPT_LOOKUP_S32(randomLimit, config->args, "-random", false, false); + PXOPT_LOOKUP_S32(min_new, config->args, "-min_new", false, false); + PXOPT_LOOKUP_F32(min_frac, config->args, "-min_frac", false, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + + if (!psListLength(where->list) && + !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString select = pxDataGet("stacktool_definebyquery_part1.sql"); + if (!select) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&select, " AND %s", whereClause); + psFree(whereClause); + } + + psString groupby = pxDataGet("stacktool_definebyquery_part2.sql"); + if (!groupby) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + psStringAppend(&select, " %s", groupby); + psFree(groupby); + + // Restriction on aggregated quantities using HAVING + { + psString havingClause = NULL; // HAVING string + if (psListLength(having->list)) { + havingClause = psDBGenerateWhereConditionSQL(having, NULL); + } + + if (min_new > 0) { + if (havingClause) { + psStringAppend(&havingClause, " AND"); + } + psStringAppend(&havingClause, + " (num_warp - num_stack >= %d OR (num_warp >= %d AND num_stack IS NULL))", + min_new, min_new); + } + if (isfinite(min_frac)) { + if (havingClause) { + psStringAppend(&havingClause, " AND"); + } + // Avoiding division by zero + psStringAppend(&havingClause, " (num_warp >= %f * num_stack OR num_stack IS NULL)", + (double)min_frac); + } + if (havingClause) { + psStringAppend(&select, " HAVING %s", havingClause); + psFree(havingClause); + } + } + psFree(having); + + if (!p_psDBRunQuery(config->dbh, select)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(select); + psFree(where); + return false; + } + psFree(select); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + psFree(where); + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + psFree(where); + return true; + } + + psString insert = NULL; // Insertion query + if (randomLimit > 0) { + insert = pxDataGet("stacktool_definebyquery_insert_random_part1.sql"); + } else { + insert = pxDataGet("stacktool_definebyquery_insert.sql"); + } + if (!insert) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&insert, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (randomLimit > 0) { + psString part2 = pxDataGet("stacktool_definebyquery_insert_random_part2.sql"); + if (!part2) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(insert); + return false; + } + psStringAppend(&insert, "%s", part2); + psFree(part2); + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + psArray *list = psArrayAllocEmpty(16); // List of runs, to print + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row from select + bool status; + + // pull out the skycell_id, tess_id, filter + psString skycell_id = psMetadataLookupStr(&status, row, "skycell_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup skycell_id"); + psFree(output); + psFree(insert); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psString tess_id = psMetadataLookupStr(&status, row, "tess_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup tess_id"); + psFree(output); + psFree(insert); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psString filter = psMetadataLookupStr(&status, row, "filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup filter"); + psFree(output); + psFree(insert); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + // create a new stackRun for this stack + stackRunRow *run = stackRunRowAlloc( + 0, // ID + "new", // state + workdir, + label, + reduction, + dvodb, + registered, + skycell_id, + tess_id, + filter); + + if (!stackRunInsertObject(config->dbh, run)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + psFree(run); + psFree(insert); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + // figure out the new stack_id + psS64 stack_id = psDBLastInsertID(config->dbh); + run->stack_id = stack_id; + + psArrayAdd(list, list->n, run); + psFree(run); + + // Create a suitable insertion query for this run + psString thisInsert = psStringCopy(insert); + psString idString = NULL; + psStringAppend(&idString, "%" PRId64, stack_id); + psStringSubstitute(&thisInsert, idString, "@STACK_ID@"); + psFree(idString); + + // XXX this insert uses a select to generate the list of warp_ids for the stack, + // we have applied a set of criteria above (WHERE) to select the relevant warps + // this insert needs to use exactly the same restrictions (race condition is probably not critical) + // the insert below seems to only restrict matches to the skycell, tess, and filter + if ((randomLimit > 0 && !p_psDBRunQuery(config->dbh, thisInsert, skycell_id, filter, randomLimit)) || + (randomLimit <= 0 && !p_psDBRunQuery(config->dbh, thisInsert, skycell_id, filter))) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(thisInsert); + psFree(insert); + psFree(output); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(thisInsert); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(list); + return false; + } + + if (!stackRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + psFree(list); + return true; +} + + +static bool definerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); + PXOPT_LOOKUP_STR(filter, config->args, "-filter", true, false); + + // default + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + + // options + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + + // we have to support multipe exp_ids + psMetadataItem *warp_ids = psMetadataLookup(config->args, "-warp_id"); + if (!warp_ids) { + // this shouldn't actually happen when using psArgs + psError(PS_ERR_UNKNOWN, true, "-warp_id is required"); + return false; + } + + stackRunRow *run = stackRunRowAlloc( + 0, // ID + "new", // state + workdir, + label, + reduction, + dvodb, + registered, + skycell_id, + tess_id, + filter); + + if (!run) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc stackRun object"); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!stackRunInsertObject(config->dbh, run)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return true; + } + + // get the assigned warp_id + run->stack_id = psDBLastInsertID(config->dbh); + + // insert the stackInputSkyfile rows + psListIterator *iter = psListIteratorAlloc(warp_ids->data.list, 0, false); + psMetadataItem *item = NULL; + while ((item = psListGetAndIncrement(iter))) { + // if the value is NULL this is probably the first pass through the + // loop and -warp_id was not specified at all + if (!item->data.V) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -warp_id"); + return false; + } + if (!stackInputSkyfileInsert(config->dbh, run->stack_id, item->data.S64)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to insert stackInputSkyfile rows"); + return false; + } + } + psFree(iter); + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!stackRunPrintObject(stdout, run, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(run); + return false; + } + + psFree(run); + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + + if (state) { + // set detRun.state to state + return setstackRunState(config, stack_id, state); + } + + return true; +} + + +static bool addinputskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false); + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); + + // XXX need to validate the warp_id here + // XXX instead of validiting it here we should just use forgein key + // constrants + if (!stackInputSkyfileInsert(config->dbh, + stack_id, + warp_id + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool inputskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // XXX require at least a stack id (add better search options) + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("stacktool_inputskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "stackInputSkyfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackInputSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool tosumMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("stacktool_tosum.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackSumSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool addsumskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false); + + // optional + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F32(dtime_stack, config->args, "-dtime_stack", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // XXX need to validate the stack_id here + // XXX instead of validiting it here we should just use forgein key + // constrants + if (!stackSumSkyfileInsert(config->dbh, + stack_id, + uri, + path_base, + "full", // data_state + bg, + bg_stdev, + dtime_stack, + hostname, + good_frac, + code + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!setstackRunState(config, stack_id, "full")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to change stackRun's state"); + return false; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool sumskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("stacktool_sumskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "stackSumSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool revertsumskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "stackSumSkyfile.fault", "=="); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + int numStacks; // Number of stacks affected + + // Update state to 'new' + { + psString update = pxDataGet("stacktool_revertsumskyfile_update.sql"); + if (!update) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&update, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, update)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(update); + psFree(where); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(update); + + numStacks = psDBAffectedRows(config->dbh); + + if (numStacks < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } + + // Delete product + { + psString delete = pxDataGet("stacktool_revertsumskyfile_delete.sql"); + if (!delete) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(delete); + + if (psDBAffectedRows(config->dbh) != numStacks) { + psError(PS_ERR_UNKNOWN, true, "Updated and deleted different number of entries!"); + psFree(where); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(where); + + return true; +} + + +static bool setstackRunState(pxConfig *config, psS64 stack_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid stackRun state: %s", state); + return false; + } + + char *query = "UPDATE stackRun SET state = '%s' WHERE stack_id = %"PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, stack_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for stack_id %"PRId64, stack_id); + return false; + } + + return true; +} + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("stacktool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackPendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool pendingcleanupskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (stack_id) { + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + } + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("stacktool_pendingcleanupskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackPendingCleanupSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("stacktool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/stacktool.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/stacktool.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/stacktool.h (revision 22161) @@ -0,0 +1,43 @@ +/* + * stacktool.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef STACKTOOL_H +#define STACKTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + STACKTOOL_MODE_NONE = 0x0, + STACKTOOL_MODE_DEFINEBYQUERY, + STACKTOOL_MODE_DEFINERUN, + STACKTOOL_MODE_UPDATERUN, + STACKTOOL_MODE_ADDINPUTSKYFILE, + STACKTOOL_MODE_INPUTSKYFILE, + STACKTOOL_MODE_TOSUM, + STACKTOOL_MODE_ADDSUMSKYFILE, + STACKTOOL_MODE_SUMSKYFILE, + STACKTOOL_MODE_REVERTSUMSKYFILE, + STACKTOOL_MODE_PENDINGCLEANUPRUN, + STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, + STACKTOOL_MODE_DONECLEANUP, +} stacktoolMode; + +pxConfig *stacktoolConfig(pxConfig *config, int argc, char **argv); + +#endif // STACKTOOL_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/stacktoolConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/stacktoolConfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/stacktoolConfig.c (revision 22161) @@ -0,0 +1,207 @@ +/* + * stacktoolConfig.c + * + * Copyright (C) 2007-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "stacktool.h" + +pxConfig *stacktoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_type", 0, "search for exp_type", "object"); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", 0, "search for label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_inst", 0, "search for camera", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_telescope", 0, "search for telescope", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", 0, "search for filter", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_uri", 0, "search for uri", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-select_dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-select_dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_airmass_max", 0, "define max airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_min", 0, "define min exposure time", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_max", 0, "define max exposure time", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_min", 0, "define min solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_max", 0, "define max solar angle", NAN); + psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random", 0, "use this number of random elements", 0); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-min_num", 0, "minimum number of inputs", 0); + psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-min_new", 0, "minimum number of new inputs", 0); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-min_frac", 0, "minumum fraction of new inputs", NAN); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -definerun + psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); + psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-filter", 0, "define filter (required)", NULL); + psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-warp_id", PS_META_DUPLICATE_OK, "include this warp ID (multiple OK, required)", 0); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-stack_id", 0, "define stack ID (required)", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + + // -addinputskyfile + psMetadata *addinputskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "define stack ID (required)", 0); + psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "define warp ID (required)", 0); + + // -inputskyfile + // XXX add additional search terms + psMetadata *inputskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(inputskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddS64(inputskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddU64(inputskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -tosum + psMetadata *tosumArgs = psMetadataAlloc(); + psMetadataAddS64(tosumArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(tosumArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddS64(tosumArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddU64(tosumArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tosumArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addsumskyfile + psMetadata *addsumskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(addsumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "define stack ID (required)", 0); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-uri", 0, "define URI of file", 0); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); + psMetadataAddF64(addsumskyfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposue background", NAN); + psMetadataAddF64(addsumskyfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposue background mean stdev", NAN); + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-dtime_stack", 0, "define elapsed processing time", NAN); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-good_frac", 0, "define %% of good pixels", NAN); + psMetadataAddS16(addsumskyfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -sumskyfile + psMetadata *sumskyfileArgs= psMetadataAlloc(); + psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", NULL); + psMetadataAddU64(sumskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(sumskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertsumskyfile + psMetadata *revertsumskyfileArgs= psMetadataAlloc(); + psMetadataAddS64(revertsumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(revertsumskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddS16(revertsumskyfileArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + + // -pendingcleanuprun + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupskyfile + psMetadata *pendingcleanupskyfileArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "Define a new stackRun by searching for warp IDs", STACKTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definerun", "", STACKTOOL_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-updaterun", "", STACKTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-addinputskyfile", "", STACKTOOL_MODE_ADDINPUTSKYFILE, addinputskyfileArgs); + PXOPT_ADD_MODE("-inputskyfile", "", STACKTOOL_MODE_INPUTSKYFILE, inputskyfileArgs); + PXOPT_ADD_MODE("-tosum", "", STACKTOOL_MODE_TOSUM, tosumArgs); + PXOPT_ADD_MODE("-addsumskyfile", "", STACKTOOL_MODE_ADDSUMSKYFILE, addsumskyfileArgs); + PXOPT_ADD_MODE("-sumskyfile", "list results of stackRun", STACKTOOL_MODE_SUMSKYFILE, sumskyfileArgs); + PXOPT_ADD_MODE("-revertsumskyfile","", STACKTOOL_MODE_REVERTSUMSKYFILE, revertsumskyfileArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", STACKTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); + PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs); + PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", STACKTOOL_MODE_DONECLEANUP, donecleanupArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/warptool.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/warptool.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/warptool.c (revision 22161) @@ -0,0 +1,1594 @@ +/* + * warptool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "warptool.h" +#include "pxwarp.h" + +static psS64 definerunMode(pxConfig *config); +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool expMode(pxConfig *config); +static bool imfileMode(pxConfig *config); +static bool tooverlapMode(pxConfig *config); +static bool addoverlapMode(pxConfig *config); +static bool scmapMode(pxConfig *config); +static bool towarpedMode(pxConfig *config); +static bool addwarpedMode(pxConfig *config); +static bool warpedMode(pxConfig *config); +static bool revertwarpedMode(pxConfig *config); +static bool blockMode(pxConfig *config); +static bool maskedMode(pxConfig *config); +static bool unblockMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupwarpMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); +static bool tocleanedskyfileMode(pxConfig *config); +static bool topurgedskyfileMode(pxConfig *config); +static bool tofullskyfileMode(pxConfig *config); +static bool updateskyfileMode(pxConfig *config); + +static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile); +static bool isValidMode(pxConfig *config, const char *mode); +bool warpCompletedRuns(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = warptoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(WARPTOOL_MODE_DEFINERUN, definerunMode); + MODECASE(WARPTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(WARPTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(WARPTOOL_MODE_EXP, expMode); + MODECASE(WARPTOOL_MODE_IMFILE, imfileMode); + MODECASE(WARPTOOL_MODE_TOOVERLAP, tooverlapMode); + MODECASE(WARPTOOL_MODE_ADDOVERLAP, addoverlapMode); + MODECASE(WARPTOOL_MODE_SCMAP, scmapMode); + MODECASE(WARPTOOL_MODE_TOWARPED, towarpedMode); + MODECASE(WARPTOOL_MODE_ADDWARPED, addwarpedMode); + MODECASE(WARPTOOL_MODE_WARPED, warpedMode); + MODECASE(WARPTOOL_MODE_REVERTWARPED, revertwarpedMode); + MODECASE(WARPTOOL_MODE_BLOCK, blockMode); + MODECASE(WARPTOOL_MODE_MASKED, maskedMode); + MODECASE(WARPTOOL_MODE_UNBLOCK, unblockMode); + MODECASE(WARPTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); + MODECASE(WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupwarpMode); + MODECASE(WARPTOOL_MODE_DONECLEANUP, donecleanupMode); + MODECASE(WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileMode); + MODECASE(WARPTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileMode); + MODECASE(WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileMode); + MODECASE(WARPTOOL_MODE_UPDATESKYFILE, updateskyfileMode); + + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static psS64 definerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(fake_id, config->args, "-fake_id", true, false); // required + PXOPT_LOOKUP_STR(mode, config->args, "-mode", true, false); // required + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_BOOL(magiced, config->args, "-magiced", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // check mode + if (mode && !isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalid mode"); + return false; + } + + warpRunRow *warpRun = warpRunRowAlloc( + 0, // ID + fake_id, + mode, + "new", // state + workdir, + "dirty", // workdir_state + label, + dvodb, + tess_id, + end_stage, + registered, + magiced + ); + if (!warpRun) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc warpRun object"); + return true; + } + if (!warpRunInsertObject(config->dbh, warpRun)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpRun); + return true; + } + + // get the assigned warp_id + psS64 warp_id = psDBLastInsertID(config->dbh); + warpRun->warp_id = warp_id; + + if (!warpRunPrintObject(stdout, warpRun, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(warpRun); + return false; + } + + psFree(warpRun); + + return warp_id; +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "newExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (!psListLength(where->list) && + !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(mode, config->args, "-set_mode", true, false); // required + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_BOOL(magiced, config->args, "-magiced", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // check mode + if (mode && !isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalid mode"); + return false; + } + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("warptool_definebyquery.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, "%s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // would could do this "all in the database" if we didn't want the option + // of changing the label/reduction/expgroup/dvodb/etc. So we're pulling the + // data out so we have the option of changing these values or leaving the + // old values in place (i.e., passing the values through). + + // loop over our list of fakeRun rows + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + fakeRunRow *row = fakeRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into fakeRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxwarpQueueByFakeID(config, + row->fake_id, + workdir ? workdir : row->workdir, + label ? label : row->label, + dvodb ? dvodb : row->dvodb, + tess_id ? tess_id : row->tess_id, + end_stage ? end_stage : row->end_stage)) + { + psError(PS_ERR_UNKNOWN, false, "failed to trying to queue fake_id: %" PRId64, row->fake_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_F32(config->args, where, "-airmass_min", "airmass", ">="); + PXOPT_COPY_F32(config->args, where, "-airmass_max", "airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<"); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_max", "ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + where = NULL; + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + + if ((!state) && (!label)) { + psError(PXTOOLS_ERR_DATA, false, "parameters (-state or -label) are required"); + psFree(where); + return false; + } + + if (state) { + // set warpRun.state to state + if (!pxwarpRunSetStateByQuery(config, where, state)) { + psFree(where); + return false; + } + } + + if (label) { + // set chipRun.label to label + if (!pxwarpRunSetLabelByQuery(config, where, label)) { + psFree(where); + return false; + } + } + + psFree(where); + + return true; +} + + +static bool expMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_exp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "warpRun"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool imfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_imfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "warpRun"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpInputImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool tooverlapMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_tooverlap.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "warpRun"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool addoverlapMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(mapfile, config->args, "-mapfile", false, false); + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (code == 0) { + if (!parseAndInsertSkyCellMap(config, mapfile)) { + psError(PS_ERR_UNKNOWN, false, "failed to inject mapfile: %s into the database", mapfile); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } else { + warpSkyCellMapInsert(config->dbh, + warp_id, + NULL, // skycell_id + NULL, // tess_id + NULL, // class_id + code // fault + ); + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile) +{ + unsigned int nFail = 0; + psMetadata *skycells = psMetadataConfigRead(NULL, &nFail, mapfile, false); + if (!skycells) { + psError(PS_ERR_UNKNOWN, false, "failed to parse mapfile: %s", mapfile); + return false; + } + if (nFail) { + psError(PS_ERR_UNKNOWN, false, "there were %d errors parsing mapfile: %s", nFail, mapfile); + psFree(skycells); + return false; + } + + psMetadataItem *item = NULL; + psMetadataIterator *iter = psMetadataIteratorAlloc(skycells, 0, NULL); + while ((item = psMetadataGetAndIncrement(iter))) { + if (item->type != PS_DATA_METADATA) { + psError(PS_ERR_UNKNOWN, false, "mapfile: %s is in the wrong format", mapfile); + psFree(iter); + psFree(skycells); + return false; + } + + psMetadata *sc = item->data.md; + // this conversion isn't strictly nessicary but it's an easy way of + // validating the format + warpSkyCellMapRow *row = warpSkyCellMapObjectFromMetadata(sc); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert mapfile: %s metdata entry into a warpSkyCellMap object", mapfile); + psFree(iter); + psFree(skycells); + return false; + } + + if (!warpSkyCellMapInsertObject(config->dbh, row)) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + psFree(row); + psFree(iter); + psFree(skycells); + return false; + } + + psFree(row); + } + psFree(iter); + psFree(skycells); + + return true; +} + + +static bool scmapMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_scmap.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "warpSkyCellMap"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpSkyCellMap", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool towarpedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyCellMap.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_towarped.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpPendingSkyCell", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool addwarpedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); + + // optional + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F32(dtime_warp, config->args, "-dtime_warp", false, false); + PXOPT_LOOKUP_S32(xmin, config->args, "-xmin", false, false); + PXOPT_LOOKUP_S32(xmax, config->args, "-xmax", false, false); + PXOPT_LOOKUP_S32(ymin, config->args, "-ymin", false, false); + PXOPT_LOOKUP_S32(ymax, config->args, "-ymax", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); + PXOPT_LOOKUP_BOOL(ignore, config->args, "-ignore", false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + // we don't want to insert the last skyfile in a run but then not mark the + // run as 'stop' + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // XXX need to validate that this coresponds to an warpInputImfile + if (!warpSkyfileInsert(config->dbh, + warp_id, + skycell_id, + tess_id, + uri, + path_base, + "full", // data_state + bg, + bg_stdev, + dtime_warp, + hostname, + good_frac, + xmin, + xmax, + ymin, + ymax, + ignore, + code + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!warpCompletedRuns(config)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +bool warpCompletedRuns(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // XXX this SQL has not been broken out to into seperate files as the MYSQL + // < 5 & MYSQL 5 versions need to be kept in sync + +#undef MYSQL5 +#if MYSQL5 + // XXX at MySQL 4.1.21 (probably all of 4.1.x) chokes and dies on this + // statement as it thinks it is trying to select from the table being + // updated. The 4.1 manual says that nested sub-queries are explicited + // allowed to do this with update statements as a temporary table is + // created so that you are not actually selecting from the table you are + // modifying. + char *query = + "UPDATE warpRun\n" + " SET warpRun.state = 'stop'\n" + " WHERE\n" + " warpRun.warp_id =\n" + " (SELECT DISTINCT\n" + " warp_id\n" + " FROM\n" + " (SELECT DISTINCT\n" + " warpRun.warp_id,\n" + " warpSkyCellMap.warp_id as foo,\n" + " warpSkyfile.warp_id as bar\n" + " FROM warpRun\n" + " JOIN warpSkyCellMap\n" + " USING(warp_id)\n" + " LEFT JOIN warpSkyfile\n" + " USING(warp_id, skycell_id, tess_id)\n" + " WHERE\n" + " warpRun.state = 'new'\n" + " GROUP BY\n" + " warpRun.warp_id\n" + " HAVING\n" + " COUNT(warpSkyCellMap.warp_id) = COUNT(warpSkyfile.warp_id)\n" + " ) as Foo\n" + " )\n"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } +#else // if MYSQL5 +{ + char *query = + "CREATE TEMPORARY TABLE finished\n" + " (warp_id INT, PRIMARY KEY(warp_id)) ENGINE=MEMORY\n"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } +} + +{ + char *query = + "INSERT INTO finished\n" + " SELECT\n" + " warp_id\n" + " FROM\n" + " (SELECT DISTINCT\n" + " warpRun.warp_id,\n" + " warpSkyCellMap.warp_id as foo,\n" + " warpSkyfile.warp_id as bar\n" + " FROM warpRun\n" + " JOIN warpSkyCellMap\n" + " USING(warp_id)\n" + " LEFT JOIN warpSkyfile\n" + " USING(warp_id, skycell_id)\n" + " WHERE\n" + " warpRun.state = 'new'\n" + " GROUP BY\n" + " warpRun.warp_id\n" + " HAVING\n" + " COUNT(warpSkyCellMap.warp_id) = COUNT(warpSkyfile.warp_id)\n" + " ) as Foo \n"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } +} + +{ + char *query = + "UPDATE warpRun\n" + " SET warpRun.state = 'full'\n" + " WHERE\n" + " warpRun.warp_id =\n" + " (SELECT DISTINCT\n" + " warp_id\n" + " FROM finished\n" + " )\n"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } +} +#endif // if MYSQL5 + + return true; +} + +static bool warpedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyfile.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyfile.tess_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_warped.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool revertwarpedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyfile.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyfile.tess_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "rawExp.reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + + PXOPT_COPY_F32(config->args, where, "-airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F32(config->args, where, "-airmass_max", "rawExp.airmass", "<"); + PXOPT_COPY_F64(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_F64(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_F64(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_F64(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_F32(config->args, where, "-solang_min", "rawExp.solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "rawExp.solang", "<"); + PXOPT_COPY_S16(config->args, where, "-code", "warpSkyfile.fault", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("warptool_revertwarped.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + if (!warpMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM warpMask"); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("warpool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + char *query = "DELETE FROM warpMask WHERE label = '%s'"; + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("warptool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpPendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool pendingcleanupwarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (warp_id) { + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + } + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("warptool_pendingcleanupskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpPendingCleanupWarp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("warptool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool isValidMode(pxConfig *config, const char *mode) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(mode, false); + + // check that state is a valid string value + if (!( + (strncmp(mode, "warp", 5) == 0) + || (strncmp(mode, "diff", 5) == 0) + || (strncmp(mode, "stack", 6) == 0) + || (strncmp(mode, "magic", 6) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid warpRun mode: %s", mode); + return false; + } + + return true; +} + +// update warpSkyfile.data_state to given value. +// afterwards, if all skfyiles in the run have the new state, update the state for the run as well +// shared code for the modes -tocleanedskyfile -tofullskyfile -topurgedskyfile + +static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // warp_id, skycell_id are required + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + + psString query = pxDataGet("warptool_change_skyfile_data_state.sql"); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // note only updates if warpRun.state = run_state + if (!p_psDBRunQuery(config->dbh, query, data_state, warp_id, skycell_id, run_state)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + query = pxDataGet("warptool_change_run_state.sql"); + if (!p_psDBRunQuery(config->dbh, query, data_state, warp_id, data_state)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} +static bool tocleanedskyfileMode(pxConfig *config) +{ + return change_skyfile_data_state(config, "cleaned", "goto_cleaned"); +} +static bool tofullskyfileMode(pxConfig *config) +{ + return change_skyfile_data_state(config, "full", "update"); +} +static bool topurgedskyfileMode(pxConfig *config) +{ + return change_skyfile_data_state(config, "purged", "goto_purged"); +} + +static bool updateskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // warp_id, skycell_id, code are required + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + psString query = pxDataGet("warptool_updateskyfile.sql"); + + // note only updates if warpRun.state = run_state + if (!p_psDBRunQuery(config->dbh, query, code, warp_id, skycell_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + return true; +} Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/warptool.h =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/warptool.h (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/warptool.h (revision 22161) @@ -0,0 +1,55 @@ +/* + * warptool.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef WARPTOOL_H +#define WARPTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + WARPTOOL_MODE_NONE = 0x0, + WARPTOOL_MODE_DEFINEBYQUERY, + WARPTOOL_MODE_DEFINERUN, + WARPTOOL_MODE_RUNONE, + WARPTOOL_MODE_UPDATERUN, + WARPTOOL_MODE_ADDINPUTEXP, + WARPTOOL_MODE_EXP, + WARPTOOL_MODE_IMFILE, + WARPTOOL_MODE_TOOVERLAP, + WARPTOOL_MODE_ADDOVERLAP, + WARPTOOL_MODE_SCMAP, + WARPTOOL_MODE_TOWARPED, + WARPTOOL_MODE_ADDWARPED, + WARPTOOL_MODE_WARPED, + WARPTOOL_MODE_REVERTWARPED, + WARPTOOL_MODE_BLOCK, + WARPTOOL_MODE_MASKED, + WARPTOOL_MODE_UNBLOCK, + WARPTOOL_MODE_PENDINGCLEANUPRUN, + WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, + WARPTOOL_MODE_DONECLEANUP, + WARPTOOL_MODE_TOCLEANEDSKYFILE, + WARPTOOL_MODE_TOPURGEDSKYFILE, + WARPTOOL_MODE_TOFULLSKYFILE, + WARPTOOL_MODE_UPDATESKYFILE, +} warptoolMode; + +pxConfig *warptoolConfig(pxConfig *config, int argc, char **argv); + +#endif // WARPTOOL_H Index: /branches/eam_branches/eam_branch_20080918/ippTools/src/warptoolConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20080918/ippTools/src/warptoolConfig.c (revision 22161) +++ /branches/eam_branches/eam_branch_20080918/ippTools/src/warptoolConfig.c (revision 22161) @@ -0,0 +1,405 @@ +/* + * warptoolConfig.c + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "warptool.h" + +pxConfig *warptoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -definebyquery + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-fake_id", 0, "search by fake_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ra_min", 0, "define min", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ra_max", 0, "define max", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-decl_min", 0, "define min", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-decl_max", 0, "define max", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define min fraction of saturated pixels", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_min", 0, "define max", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_max", 0, "define max", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "define max", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define max", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-alt_min", 0, "define min", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-alt_max", 0, "define max", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-az_min", 0, "define min", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-az_max", 0, "define max", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "search on fakeRun label", NULL); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_mode", 0, "define mode (warp, diff, stack, magic)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tess ID", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-magiced", 0, "has this exposure been magiced", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -definerun + psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-fake_id", 0, "define camtool ID (required)", 0); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-mode", 0, "define mode (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tess_id", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-end_stage", 0, "define end stage", NULL); + psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-magiced", 0, "has this exposure been magiced", false); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updaterun + // XXX need to allow multiple fake_ids + // XXX need to allow multiple fake_ids + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-fake_id", 0, "search by fake_id", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ra_min", 0, "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ra_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-decl_min", 0, "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-decl_max", 0, "define max", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define min fraction of saturated pixels", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_min", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-alt_min", 0, "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-alt_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-az_min", 0, "define min", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-az_max", 0, "define max", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "set label", NULL); + +#if 0 + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); +#endif + + // -exp + psMetadata *expArgs = psMetadataAlloc(); + psMetadataAddS64(expArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddS64(expArgs, PS_LIST_TAIL, "-fake_id", 0, "search by camtool ID", 0); + psMetadataAddU64(expArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(expArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -imfile + psMetadata *imfileArgs = psMetadataAlloc(); + psMetadataAddS64(imfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddS64(imfileArgs, PS_LIST_TAIL, "-fake_id", 0, "search by camtool ID", 0); + psMetadataAddU64(imfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(imfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -tooverlap + psMetadata *tooverlapArgs = psMetadataAlloc(); + psMetadataAddS64(tooverlapArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddStr(tooverlapArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddU64(tooverlapArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tooverlapArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addoverlap + psMetadata *addoverlapArgs = psMetadataAlloc(); + psMetadataAddStr(addoverlapArgs, PS_LIST_TAIL, "-mapfile", 0, "path to skycell <-> imfile mapping file", NULL); + psMetadataAddS64(addoverlapArgs, PS_LIST_TAIL, "-warp_id", 0, "set warp ID", 0); + psMetadataAddS16(addoverlapArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -scmap + psMetadata *scmapArgs = psMetadataAlloc(); + psMetadataAddS64(scmapArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddStr(scmapArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddStr(scmapArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL); + psMetadataAddU64(scmapArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(scmapArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -towarped + psMetadata *towarpedArgs = psMetadataAlloc(); + psMetadataAddS64(towarpedArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddStr(towarpedArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddU64(towarpedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(towarpedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addwarped + psMetadata *addwarpedArgs = psMetadataAlloc(); + psMetadataAddS64(addwarpedArgs, PS_LIST_TAIL, "-warp_id", 0, "define warptool ID (required)", 0); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID (required)", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID (required)", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-uri", 0, "define URI of file", 0); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); + psMetadataAddF64(addwarpedArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addwarpedArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-dtime_warp", 0, "define elapsed processing time", NAN); + psMetadataAddS32(addwarpedArgs, PS_LIST_TAIL, "-xmin", 0, "define minimum x value", INT_MAX); + psMetadataAddS32(addwarpedArgs, PS_LIST_TAIL, "-xmax", 0, "define maximum x value", -INT_MAX); + psMetadataAddS32(addwarpedArgs, PS_LIST_TAIL, "-ymin", 0, "define minimum y value", INT_MAX); + psMetadataAddS32(addwarpedArgs, PS_LIST_TAIL, "-ymax", 0, "define maximum y value", -INT_MAX); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-good_frac", 0, "define %% of good pixels", NAN); + psMetadataAddBool(addwarpedArgs, PS_LIST_TAIL, "-ignore", 0, "define if this skycell should be ignored", false); + psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -warped + psMetadata *warpedArgs = psMetadataAlloc(); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID", NULL); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID", NULL); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure tag", 0); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure tag", 0); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-fake_id", 0, "define phase 3 version of exposure tag", 0); + psMetadataAddU64(warpedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertwarped + // XXX need to allow multiple fake_ids + // XXX need to allow multiple fake_ids + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *revertwarpedArgs = psMetadataAlloc(); + psMetadataAddS64(revertwarpedArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-tess_id", 0, "searcy by tessellation ID", NULL); + psMetadataAddS64(revertwarpedArgs, PS_LIST_TAIL, "-fake_id", 0, "search by fake_id", 0); + psMetadataAddS64(revertwarpedArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); + psMetadataAddS64(revertwarpedArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(revertwarpedArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(revertwarpedArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-ra_min", 0, "define min", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-ra_max", 0, "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-decl_min", 0, "define min", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-decl_max", 0, "define max", NAN); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-exp_time_min", 0, "define min", NAN); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-exp_time_max", 0, "define max", NAN); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define min fraction of saturated pixels", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_min", 0, "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_max", 0, "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-alt_min", 0, "define min", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-alt_max", 0, "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-az_min", 0, "define min", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-az_max", 0, "define max", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); + psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + + psMetadataAddS16(revertwarpedArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0); + psMetadataAddBool(revertwarpedArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + + // -pendingcleanuprun + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupskyfile + psMetadata *pendingcleanupskyfileArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -tocleanedskyfile + psMetadata *tocleanedskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanedskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); + psMetadataAddStr(tocleanedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + + // -topurgedskyfile + psMetadata *topurgedskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(topurgedskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); + psMetadataAddStr(topurgedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + + // -tofullskyfile + psMetadata *tofullskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); + psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + + // -toupdateskyfile + psMetadata *updateskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(updateskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); + psMetadataAddStr(updateskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + psMetadataAddS16(updateskyfileArgs, PS_LIST_TAIL, "-code", 0, "new fault code", 0); + + psFree(now); + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "", WARPTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definerun", "", WARPTOOL_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-updaterun", "", WARPTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-exp", "", WARPTOOL_MODE_EXP, expArgs); + PXOPT_ADD_MODE("-imfile", "", WARPTOOL_MODE_IMFILE, imfileArgs); + PXOPT_ADD_MODE("-tooverlap", "", WARPTOOL_MODE_TOOVERLAP, tooverlapArgs); + PXOPT_ADD_MODE("-addoverlap", "", WARPTOOL_MODE_ADDOVERLAP, addoverlapArgs); + PXOPT_ADD_MODE("-scmap", "", WARPTOOL_MODE_SCMAP, scmapArgs); + PXOPT_ADD_MODE("-towarped", "", WARPTOOL_MODE_TOWARPED, towarpedArgs); + PXOPT_ADD_MODE("-addwarped", "", WARPTOOL_MODE_ADDWARPED, addwarpedArgs); + PXOPT_ADD_MODE("-warped", "", WARPTOOL_MODE_WARPED, warpedArgs); + PXOPT_ADD_MODE("-revertwarped", "", WARPTOOL_MODE_REVERTWARPED, revertwarpedArgs); + PXOPT_ADD_MODE("-block", "set a label block", WARPTOOL_MODE_BLOCK, blockArgs); + PXOPT_ADD_MODE("-masked", "show blocked lables", WARPTOOL_MODE_MASKED, maskedArgs); + PXOPT_ADD_MODE("-unblock", "remove a label block", WARPTOOL_MODE_UNBLOCK, unblockArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", WARPTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); + PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs); + PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", WARPTOOL_MODE_DONECLEANUP, donecleanupArgs); + PXOPT_ADD_MODE("-tocleanedskyfile", "set skyfile as cleaned", WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileArgs); + PXOPT_ADD_MODE("-topurgedskyfile", "set skyfile as purged", WARPTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileArgs); + PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full (updated)", WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs); + PXOPT_ADD_MODE("-updateskyfile", "update fault code for skyfile", WARPTOOL_MODE_UPDATESKYFILE, updateskyfileArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +}