Index: /branches/eam_branches/eam_branch_20080806/dbconfig/.cvsignore =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/.cvsignore (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/.cvsignore (revision 22160) @@ -0,0 +1,2 @@ +ippdb.m4 +ippdb.mdc Index: /branches/eam_branches/eam_branch_20080806/dbconfig/Makefile =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/Makefile (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/Makefile (revision 22160) @@ -0,0 +1,26 @@ +SHELL=/bin/sh +GLUEFORGE=`which glueforge` + +all: ippdb.mdc + +ippdb.mdc : ipp.m4 *.md + m4 ipp.m4 > ippdb.mdc + +## please leave the output target as ippdb.src and +## move to ippdb by hand for cvs import +install: ippdb.mdc + rm -rf ../ippdb.src + $(GLUEFORGE) -i ippdb.mdc --output ../ippdb.src + chmod +x ../ippdb.src/autogen.sh + +src: ippdb.mdc + $(GLUEFORGE) -i ippdb.mdc --output ../ippdb + chmod +x ../ippdb/autogen.sh + +foo: src + $(MAKE) -C ../ippdb install + +build : ippdb ippdb.mdc + +clean: + @rm -f *~ Index: /branches/eam_branches/eam_branch_20080806/dbconfig/calibration.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/calibration.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/calibration.md (revision 22160) @@ -0,0 +1,15 @@ + +# table of the DVO databases which get calibrated +calDB METADATA + cal_id S64 0 # Primary Key AUTO_INCREMENT + dvodb STR 64 + state STR 64 +END + +calRun METADATA + cal_id S64 0 # Primary Key AUTO_INCREMENT + region STR 64 # success or failure + last_step STR 64 # fkey(cal_id) ref calDB(cal_id) + state STR 64 # +END + Index: /branches/eam_branches/eam_branch_20080806/dbconfig/cam.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/cam.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/cam.md (revision 22160) @@ -0,0 +1,50 @@ +camRun METADATA + cam_id S64 0 # Primary Key AUTO_INCREMENT + chip_id S64 0 # Key INDEX(cam_id, chip_id) fkey(chip_id) ref chipRun(chip_id) + state STR 64 # key + workdir STR 255 + workdir_state STR 64 # key + label STR 64 # key + reduction STR 64 + expgroup STR 64 # key + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key +END + +camProcessedExp METADATA +# the camPendingExp row gets deleted so we can not put a fkey on cam_id + cam_id S64 0 # Primary Key + uri STR 255 # fkey(cam_id) ref camRun(cam_id) + + bg F32 0.0 + bg_stdev F32 0.0 + bg_mean_stdev F32 0.0 + bias F32 0.0 + bias_stdev F32 0.0 + fringe_0 F32 0.0 + fringe_1 F32 0.0 + fringe_2 F32 0.0 + sigma_ra F32 0.0 + sigma_dec F32 0.0 + ap_resid F32 0.0 + ap_resid_stdev F32 0.0 + zp_mean F32 0.0 + zp_stdev F32 0.0 + fwhm_major F32 0.0 + fwhm_minor F32 0.0 + dtime_detrend F32 0.0 + dtime_photom F32 0.0 + dtime_astrom F32 0.0 + hostname STR 64 + n_stars S32 0 + n_extended S32 0 + n_cr S32 0 + n_astrom S32 0 + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +camMask METADATA + label STR 64 # Primary Key +END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/changes.txt =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/changes.txt (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/changes.txt (revision 22160) @@ -0,0 +1,366 @@ +-- How to manually change the database between versions via mySQL commands. +-- +-- This file is generated manually, and may not be complete. + + +-- Version 1.1.20 --> 1.1.21: + +alter table camProcessedExp drop nastro; +alter table camProcessedExp add (fwhm double, fwhm_range double, n_stars int(11), n_extended int(11), n_cr int(11), n_astrom int(11)); +alter table chipProcessedImfile add column (bias double, bias_stdev double, fringe_0 double, fringe_1 double, fringe_2 double, sigma_ra double, sigma_dec double, ap_resid double, ap_resid_stdev double, fwhm double, fwhm_range double, n_stars int(11), n_extended int(11), n_cr int(11), n_astrom int(11)); +alter table warpSkyfile add column path_base varchar(255) NULL DEFAULT NULL after uri; +alter table diffSkyfile add column path_base varchar(255) NULL DEFAULT NULL after uri; +alter table stackSumSkyfile add column path_base varchar(255) NULL DEFAULT NULL after uri; + + +-- Version 1.1.22 --> 1.1.23 +-- Adding support for reduction classes (which are used to specify recipes for parts of the pipeline). +# Adding 'reduction' to detRun +# Renaming 'recipe' in {chip,cam}{Pending,Processed}Exp to 'reduction' + +alter table detRun add column reduction varchar(64) NULL DEFAULT NULL after exp_type; +alter table chipPendingExp change column recipe reduction varchar(64); +alter table chipProcessedExp change column recipe reduction varchar(64); +alter table camPendingExp change column recipe reduction varchar(64); +alter table camProcessedExp change column recipe reduction varchar(64); + +-- Version ??? --> 1.1.29 + +alter table warpSkyfile add column ignored tinyint after good_frac; + +-- +-- The following set of changes were applied to update the gpc1 database on 2008/02/07 +-- + +alter table calDB change column catdir dvodb varchar(64); + +alter table camProcessedExp change column fwhm fwhm_major float; +alter table camProcessedExp change column fwhm_range fwhm_minor float; + +-- In order to keep the column order correct this is split up below +--alter table camProcessedExp add column ( bias float, bias_stdev float, fringe_0 float, fringe_1 float, fringe_2 float, ap_resid float, ap_resid_stdev float, dtime_detrend float, dtime_photom float, dtime_astrom float, hostname varchar(64) ); + +alter table camProcessedExp add column bias float after bg_mean_stdev; +alter table camProcessedExp add column bias_stdev float after bias; +alter table camProcessedExp add column fringe_0 float after bias_stdev; +alter table camProcessedExp add column fringe_1 float after fringe_0; +alter table camProcessedExp add column fringe_2 float after fringe_1; +alter table camProcessedExp add column ap_resid float after sigma_dec; +alter table camProcessedExp add column ap_resid_stdev float after ap_resid; +alter table camProcessedExp add column dtime_detrend float after fwhm_minor; +alter table camProcessedExp add column dtime_photom float after dtime_detrend; +alter table camProcessedExp add column dtime_astrom float after dtime_photom; +alter table camProcessedExp add column hostname varchar(64) after dtime_astrom; + + +alter table chipProcessedImfile change column fwhm fwhm_major float; +alter table chipProcessedImfile change column fwhm_range fwhm_minor float; + +-- In order to keep the column order correct this is split up below +-- alter table chipProcessedImfile add column ( zp_mean float, zp_stdev float, dtime_detrend float, dtime_photom float, dtime_astrom float, hostname varchar(64) ); + +alter table chipProcessedImfile add column zp_mean float after ap_resid_stdev; +alter table chipProcessedImfile add column zp_stdev float after zp_mean; +alter table chipProcessedImfile add column dtime_detrend float after fwhm_minor; +alter table chipProcessedImfile add column dtime_photom float after dtime_detrend; +alter table chipProcessedImfile add column dtime_astrom float after dtime_photom; +alter table chipProcessedImfile add column hostname varchar(64) after dtime_astrom; + + +-- In order to keep the column order correct this is split up below +-- alter table diffSkyfile add column ( dtime_diff float, hostname varchar(64) ); +alter table diffSkyfile add column dtime_diff float after bg_stdev; +alter table diffSkyfile add column hostname varchar(64) after dtime_diff; +alter table diffSkyfile change column good_frac good_frac float; + +alter table flatcorrRun add column filter varchar(64) after dvodb; + +alter table stackSumSkyfile change column good_frac good_frac float; + +-- alter table stackSumSkyfile add column ( dtime_stack float, hostname varchar(64) ); +alter table stackSumSkyfile add column dtime_stack float after bg_stdev; +alter table stackSumSkyfile add column hostname varchar(64) after dtime_stack; + +#alter table warpSkyfile add column ( dtime_warp float, hostname varchar(64) ); +alter table warpSkyfile add column dtime_warp float after bg_stdev; +alter table warpSkyfile add column hostname varchar(64) after dtime_warp; + +alter table warpSkyfile change column good_frac good_frac float; + +alter table newExp add column dvodb varchar(255) after reduction; +alter table newExp add column tess_id varchar(64) after dvodb; +alter table rawExp add column dvodb varchar(255) after reduction; +alter table rawExp add column tess_id varchar(64) after dvodb; +alter table chipRun add column tess_id varchar(64) after dvodb; +alter table camRun add column tess_id varchar(64) after dvodb; +alter table warpRun add column tess_id varchar(64) after dvodb; + +alter table camRun add column end_stage varchar(64) after tess_id; +alter table chipRun add column end_stage varchar(64) after tess_id; +alter table warpRun add column end_stage varchar(64) after tess_id; +alter table newExp add column end_stage varchar(64) after tess_id; +alter table rawExp add column end_stage varchar(64) after tess_id; +alter table camRun add key (end_stage); +alter table chipRun add key (end_stage); +alter table warpRun add key (end_stage); +alter table newExp add key (end_stage); +alter table rawExp add key (end_stage); + +alter table warpRun add column workdir_state varchar(64) after workdir; +alter table warpRun add key (workdir_state); + +alter table pzDoneImfile add column fault smallint(6); + +-- end of changes to 1.1.29 + +alter table newExp add column label varchar(64) after end_stage; + +-- 1.1.30 + +drop table pzDoneExp; +drop table pzDoneImfile; +drop table pzPendingExp; +drop table pzPendingImfile; +alter table flatcorrRun add column region VARCHAR(255) after stats; + +-- 1.1.31 + +ALTER TABLE warpRun ADD COLUMN cam_id BIGINT AFTER warp_id; +ALTER TABLE warpRun ADD KEY(cam_id); +ALTER TABLE warpRun ADD KEY(warp_id, cam_id); +ALTER TABLE warpRun ADD CONSTRAINT FOREIGN KEY(cam_id) REFERENCES +camRun(cam_id); +ALTER TABLE warpRun ADD COLUMN magiced TINYINT AFTER registered; +ALTER TABLE warpSkyCellMap DROP FOREIGN KEY warpSkyCellMap_ibfk_1; +UPDATE warpRun JOIN warpInputExp USING(warp_id) SET warpRun.cam_id = +warpInputExp.cam_id WHERE warpRun.warp_id = warpInputExp.warp_id; +ALTER TABLE warpSkyCellMap ADD CONSTRAINT FOREIGN KEY(warp_id, cam_id) +REFERENCES warpRun(warp_id, cam_id); +DROP TABLE warpInputExp; + +-- 1.1.32 +ALTER TABLE warpRun ADD COLUMN label VARCHAR(64) AFTER workdir_state; +ALTER TABLE warpRun ADD KEY(label); +CREATE TABLE warpMask (label VARCHAR(64), PRIMARY KEY(label)) ENGINE=innodb; + +-- 1.1.33 +ALTER TABLE summitExp ADD COLUMN fault smallint(6) NOT NULL AFTER imfiles; +ALTER TABLE summitExp ADD KEY(fault); + +-- 1.1.34 +ALTER TABLE rawImfile ADD KEY(exp_name); + +-- 1.1.35 +ALTER TABLE rawImfile ADD INDEX UNQIUE(exp_id, tmp_class_id); + +-- 1.1.36 +ALTER TABLE warpSkyfile ADD COLUMN (xmin INT, xmax INT, ymin INT, ymax INT); +ALTER TABLE diffSkyfile ADD COLUMN (stamps_num INT, stamps_rms FLOAT, sources INT); +ALTER TABLE rawImfile ADD COLUMN hostname VARCHAR(64) AFTER object; +ALTER TABLE rawExp ADD COLUMN hostname VARCHAR(64) AFTER object; + +-- 1.1.37 +ALTER TABLE pzDataStore ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER uri; +ALTER TABLE summitExp ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER fault; +ALTER TABLE summitImfile ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER uri; +ALTER TABLE pzDownloadExp ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER state; +ALTER TABLE pzDownloadImfile ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER fault; +ALTER TABLE newExp ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER label; +ALTER TABLE newImfile ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER uri; +ALTER TABLE rawExp ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER fault; +ALTER TABLE rawImfile ADD COLUMN epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP AFTER fault; + +-- 1.1.38 + +-- convert from utf8 -> latin1 for performance/space gains +SET FOREIGN_KEY_CHECKS=0; +ALTER TABLE calDB CONVERT TO CHARACTER SET latin1; +ALTER TABLE calRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE camMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE camProcessedExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE camRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE chipMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE chipProcessedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE chipRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE detInputExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE detNormalizedExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE detNormalizedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detNormalizedStatImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detProcessedExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE detProcessedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detRegisteredImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detResidExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE detResidImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE detRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE detRunSummary CONVERT TO CHARACTER SET latin1; +ALTER TABLE detStackedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE diffInputSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE diffRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE diffSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE fakeMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE fakeProcessedImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE fakeRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE flatcorrExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE flatcorrRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE guidePendingExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicInputSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicNodeResult CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicSkyfileMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE magicTree CONVERT TO CHARACTER SET latin1; +ALTER TABLE newExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE newImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE pstampDataStore CONVERT TO CHARACTER SET latin1; +ALTER TABLE pstampJob CONVERT TO CHARACTER SET latin1; +ALTER TABLE pstampRequest CONVERT TO CHARACTER SET latin1; +ALTER TABLE pzDataStore CONVERT TO CHARACTER SET latin1; +ALTER TABLE pzDownloadExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE pzDownloadImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE rawExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE rawImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE stackInputSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE stackRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE stackSumSkyfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE summitExp CONVERT TO CHARACTER SET latin1; +ALTER TABLE summitImfile CONVERT TO CHARACTER SET latin1; +ALTER TABLE warpMask CONVERT TO CHARACTER SET latin1; +ALTER TABLE warpRun CONVERT TO CHARACTER SET latin1; +ALTER TABLE warpSkyCellMap CONVERT TO CHARACTER SET latin1; +ALTER TABLE warpSkyfile CONVERT TO CHARACTER SET latin1; +SET FOREIGN_KEY_CHECKS=1; + +-- first use of fake* tables +-- drop and recreate any fake* tables created prior to this point + +DROP TABLE IF EXISTS fakeRun; +CREATE TABLE fakeRun ( + fake_id BIGINT AUTO_INCREMENT, + cam_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + reduction VARCHAR(64), + expgroup VARCHAR(64), + dvodb VARCHAR(255), + tess_id VARCHAR(64), + end_stage VARCHAR(64), + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(fake_id), + KEY(cam_id), + KEY(state), + KEY(label), + KEY(expgroup), + KEY(end_stage), + INDEX(fake_id, cam_id), + FOREIGN KEY (cam_id) + REFERENCES camRun(cam_id)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +DROP TABLE IF EXISTS fakeProcessedImfile; +CREATE TABLE fakeProcessedImfile ( + fake_id BIGINT AUTO_INCREMENT, + exp_id BIGINT(20), + class_id VARCHAR(64), + uri VARCHAR(255), + dtime_fake FLOAT, + hostname VARCHAR(64), + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(fake_id, exp_id, class_id), + KEY(fault), + FOREIGN KEY (fake_id) + REFERENCES fakeRun(fake_id), + FOREIGN KEY(exp_id, class_id) + REFERENCES rawImfile(exp_id, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +DROP TABLE IF EXISTS fakeMask; +CREATE TABLE fakeMask ( + label VARCHAR(64), + PRIMARY KEY(label)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +-- re-normalize camProcessedExp table +-- unrelated to other changes +ALTER TABLE camProcessedExp DROP FOREIGN KEY camProcessedExp_ibfk_1; +ALTER TABLE camProcessedExp DROP COLUMN chip_id; +ALTER TABLE camProcessedExp ADD FOREIGN KEY (cam_id) REFERENCES camRun (cam_id); + +-- insert fakeRun betwen camRun and warpRun +-- populate fakeRun/fakeProcessedImfile +-- then heal the foreign key references + +CREATE TEMPORARY TABLE warpCamMap ( + warp_id BIGINT, + cam_id BIGINT, + PRIMARY KEY(warp_id), + KEY(cam_id) +) ENGINE=MEMORY; + +INSERT INTO warpCamMap SELECT warp_id, cam_id FROM warpRun; + +ALTER TABLE warpRun drop FOREIGN KEY warpRun_ibfk_1; +ALTER TABLE warpSkyCellMap DROP FOREIGN KEY warpSkyCellMap_ibfk_1; + +ALTER TABLE warpRun change COLUMN cam_id fake_id BIGINT(20) DEFAULT NULL; +ALTER TABLE warpSkyCellMap DROP COLUMN cam_id; + +SET FOREIGN_KEY_CHECKS=0; +ALTER TABLE warpRun ADD FOREIGN KEY (fake_id) REFERENCES fakeRun (fake_id); +SET FOREIGN_KEY_CHECKS=1; +ALTER TABLE warpSkyCellMap ADD FOREIGN KEY (warp_id) REFERENCES warpRun(warp_id); + +INSERT INTO fakeRun +SELECT + NULL, + camRun.cam_id, + 'stop', + workdir, + label, + reduction, + expgroup, + dvodb, + tess_id, + end_stage, + NULL +FROM camRun +JOIN warpCamMap + using(cam_id); + +INSERT INTO fakeProcessedImfile +SELECT + fake_id, + chipRun.exp_id, + class_id, + uri, + 0, + hostname, + path_base, + fault, + NULL +FROM fakeRun +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + USING(chip_id); + +UPDATE warpRun, fakeRun, warpCamMap +SET warpRun.fake_id = fakeRun.fake_id +WHERE warpRun.warp_id = warpCamMap.warp_id + AND warpCamMap.cam_id = fakeRun.fake_id; + +DROP TABLE warpCamMap; + + +-- new values for chipRun.state + +update chipRun set state = 'new' where state = 'run'; +update chipRun set state = 'full' where state = 'stop'; + +-- add 'data_state' to detrend tables + +alter table detRegisteredImfile add column data_state varchar(64) after path_base; Index: /branches/eam_branches/eam_branch_20080806/dbconfig/chip.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/chip.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/chip.md (revision 22160) @@ -0,0 +1,50 @@ +chipRun METADATA + chip_id S64 0 # Primary Key AUTO_INCREMENT + exp_id S64 64 # Key INDEX(chip_id, exp_id) fkey (exp_id) ref rawExp(exp_id) + state STR 64 # Key + workdir STR 255 + workdir_state STR 64 # Key + label STR 64 # Key + reduction STR 64 # Reduction class + expgroup STR 64 # Key + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key +END + +chipProcessedImfile METADATA + chip_id S64 0 # Primary Key fkey (chip_id, exp_id) ref chipRun(chip_id, exp_id) + exp_id S64 64 # Primary Key fkey (exp_id, class_id) ref rawImfile(exp_id, class_id) + class_id STR 64 # Primary Key + uri STR 255 + bg F32 0.0 + bg_stdev F32 0.0 + bg_mean_stdev F32 0.0 + bias F32 0.0 + bias_stdev F32 0.0 + fringe_0 F32 0.0 + fringe_1 F32 0.0 + fringe_2 F32 0.0 + sigma_ra F32 0.0 + sigma_dec F32 0.0 + ap_resid F32 0.0 + ap_resid_stdev F32 0.0 + zp_mean F32 0.0 + zp_stdev F32 0.0 + fwhm_major F32 0.0 + fwhm_minor F32 0.0 + dtime_detrend F32 0.0 + dtime_photom F32 0.0 + dtime_astrom F32 0.0 + hostname STR 64 + n_stars S32 0 + n_extended S32 0 + n_cr S32 0 + n_astrom S32 0 + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +chipMask METADATA + label STR 64 # Primary Key +END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/config.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/config.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/config.md (revision 22160) @@ -0,0 +1,5 @@ +glueforge METADATA + pkg_name STR ippdb + pkg_namespace STR ippdb + pkg_version STR 1.1.39 +END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/det.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/det.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/det.md (revision 22160) @@ -0,0 +1,273 @@ +detRun METADATA + det_id S64 0 # Primary Key AUTO_INCREMENT + iteration S32 0 # Key INDEX(det_id, iteration) + det_type STR 64 # Key + mode STR 64 # Key + state STR 64 # Key + filelevel STR 64 + workdir STR 255 # destination for output files + camera STR 64 + telescope STR 64 + exp_type STR 64 # XXX this should be dropped + reduction STR 64 # Reduction clas + filter STR 64 + airmass_min F32 0.0 + airmass_max F32 0.0 + exp_time_min F32 0.0 + exp_time_max F32 0.0 + ccd_temp_min F32 0.0 + ccd_temp_max F32 0.0 + posang_min F64 0.0 + posang_max F64 0.0 + registered TAI 0001-01-01T00:00:00Z + time_begin TAI 0001-01-01T00:00:00Z + time_end TAI 0001-01-01T00:00:00Z + use_begin TAI 0001-01-01T00:00:00Z + use_end TAI 0001-01-01T00:00:00Z + solang_min F32 0.0 + solang_max F32 0.0 + label STR 64 # key + parent S32 0 # Key +END + +detInputExp METADATA + det_id S64 0 # Primary Key fkey(det_id) ref detRun(det_id) + iteration S32 0 # Primary Key fkey(exp_id) ref rawExp(exp_id) + exp_id S64 64 # Primary Key INDEX(det_id, exp_id) + include BOOL f # INDEX(det_id, iteration) +END + +detProcessedImfile METADATA + det_id S64 0 # Primary Key fkey(det_id, exp_id) ref detInputExp(det_id, exp_id) + exp_id S64 64 # Primary Key fkey(exp_id, class_id) ref rawImfile(exp_id, class_id) + class_id STR 64 # Primary Key INDEX(det_id, class_id) + uri STR 255 # INDEX(det_id, exp_id) + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + fringe_0 F64 0.0 + fringe_1 F64 0.0 + fringe_2 F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +detProcessedExp METADATA + det_id S64 0 # Primary Key fkey(det_id, exp_id) ref detInputExp(det_id, exp_id) + exp_id S64 64 # Primary Key fkey(det_id, exp_id) ref detProcessedImfile(det_id, exp_id) + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + fringe_0 F64 0.0 + fringe_1 F64 0.0 + fringe_2 F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +# +# detStackedImfile does not depend on detProcessedExp. detProcesedExp is purely +# FYI values and can be calculated in parellel +# +detStackedImfile METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration) ref detInputExp(det_id, iteration) + iteration S32 0 # Primary Key fkey(det_id, class_id) ref detProcessedImfile(det_id, class_id) + class_id STR 64 # Primary Key + uri STR 255 + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 +# XXX does it make sense to 'clean' the stacked imfiled? + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +detNormalizedStatImfile METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration) ref detInputExp(det_id, iteration) + iteration S32 0 # Primary Key fkey(det_id, iteration, class_id) ref detStackedImfile(det_id, iteration, class_id) + class_id STR 64 # Primary Key + norm F32 0.0 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +## XXX this must match in fields with detRegisteredImfile +detNormalizedImfile METADATA + det_id S64 0 # Primary Key fkey(det_id) ref detInputExp(det_id) + iteration S32 0 # Primary Key fkey(det_id, iteration, class_id) ref detNormalizedStatImfile(det_id, iteration, class_id) + class_id STR 64 # Primary Key INDEX(det_id, iteration) + uri STR 255 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +detNormalizedExp METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration) ref detInputExp(det_id, iteration) + iteration S32 0 # Primary Key fkey(det_id, iteration) ref detNormalizedImfile(det_id, iteration) + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +#detMasterFrame METADATA +# det_id S64 0 # Primary Key +# iteration S32 0 # Primary Key +# comment STR 255 +#END +# +## drop? +#detMasterImfile METADATA +# det_id S64 0 # Primary Key +# class_id STR 64 # Primary Key +# uri STR 255 +# recipe STR 64 +#END + +detResidImfile METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration, exp_id) ref detInputExp(det_id, iteration, exp_id) + iteration S32 0 # Primary Key fkey(det_id, exp_id, class_id) ref detProcessedImfile(det_id, exp_id, class_id) + exp_id S64 64 # Primary Key fkey(det_id, iteration) ref detNormalizedExp(det_id, iteration) + class_id STR 64 # Primary Key INDEX(det_id, iteration, exp_id) + uri STR 255 + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + bg_skewness F64 0.0 + bg_kurtosis F64 0.0 + bin_stdev F64 0.0 + fringe_0 F64 0.0 + fringe_1 F64 0.0 + fringe_2 F64 0.0 + fringe_resid_0 F64 0.0 + fringe_resid_1 F64 0.0 + fringe_resid_2 F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +detResidExp METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration, exp_id) ref detInputExp(det_id, iteration, exp_id) + iteration S32 0 # Primary Key fkey(det_id, iteration, exp_id) ref detResidImfile(det_id, iteration, exp_id) + exp_id S64 64 # Primary Key INDEX(det_id, iteration) + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + bg_skewness F64 0.0 + bg_kurtosis F64 0.0 + bin_stdev F64 0.0 + fringe_0 F64 0.0 + fringe_1 F64 0.0 + fringe_2 F64 0.0 + fringe_resid_0 F64 0.0 + fringe_resid_1 F64 0.0 + fringe_resid_2 F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + accept BOOL f + fault S16 0 # Key NOT NULL +END + +# XXX this probably should have been defined at the start of a new iteration, not at the end. +detRunSummary METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration) ref detInputExp(det_id, iteration) + iteration S32 0 # Primary Key fkey(det_id, iteration) ref detResidExp(det_id, iteration) + data_state STR 64 # full, goto_cleaned, cleaned, goto_full, goto_purged, purged + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + accept BOOL f + fault S16 0 # Key NOT NULL +END + +# +# Note: This table needs to stay more or less identical to detNormalizedImfile. +# It only exists as a seperate entity so they it can have different fkeys +# +detRegisteredImfile METADATA + det_id S64 0 # Primary Key fkey(det_id, iteration) ref detRun(det_id, iteration) + iteration S32 0 # Primary Key + class_id STR 64 # Primary Key + uri STR 255 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + data_state STR 64 # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration) + fault S16 0 # Key NOT NULL +END + +detCorrectedExp METADATA + det_id S64 0 # Primary Key fkey(det_id, exp_id) ref detInputExp(det_id, exp_id) + exp_id S64 64 # Primary Key fkey(exp_id, class_id) ref rawImfile(exp_id, class_id) + uri STR 255 # INDEX(det_id, exp_id) + corr_id S64 0 + corr_type STR 64 + recipe STR 64 + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +detCorrectedImfile METADATA + det_id S64 0 # Primary Key fkey(det_id, exp_id) ref detInputExp(det_id, exp_id) + exp_id S64 64 # Primary Key fkey(exp_id, class_id) ref rawImfile(exp_id, class_id) + class_id STR 64 # Primary Key INDEX(det_id, class_id) + uri STR 255 # INDEX(det_id, exp_id) + path_base STR 255 + fault S16 0 # Key NOT NULL +END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/diff.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/diff.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/diff.md (revision 22160) @@ -0,0 +1,42 @@ +# $Id: diff.md,v 1.9 2008-05-24 00:41:13 price Exp $ + +diffRun METADATA + diff_id S64 0 # Primary Key AUTO_INCREMENT + state STR 64 # Key + workdir STR 255 + dvodb STR 255 + registered TAI NULL + skycell_id STR 64 # Key + tess_id STR 64 # Key +END + +# +# Diff Sky Cells Mode +# + +# only ever 2 per run - one template / one not +diffInputSkyfile METADATA + diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) + template BOOL f # Primary Key + stack_id S64 0 # fkey(stack_id) ref stackSumSkyfile(stack_id) + warp_id S64 0 # fkey(warp_id, skycell_id, tess_id) ref warpSkyfile(warp_id, skycell_id, tess_id) + skycell_id STR 64 # Key + tess_id STR 64 # Key +# either a input or a template + kind STR 64 # Key +END + +diffSkyfile METADATA + diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) + uri STR 255 + path_base STR 255 + bg F64 0.0 + bg_stdev F64 0.0 + stamps_num S32 0 + stamps_rms F32 0.0 + sources S32 0 + dtime_diff F32 0.0 + hostname STR 64 + good_frac F32 0.0 # Key + fault S16 0 # Key +END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/dimm.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/dimm.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/dimm.md (revision 22160) @@ -0,0 +1,21 @@ +# Table 13: DIMM Measurements Table +# Column Name Datatype Description +# Time date/time The time the DIMM observation was taken. +# sigmax float Raw dispersion in x. +# sigmay float Raw dispersion in y. +# FWHM float Dervied seeing full width at half maximum. +# RA float The coordinates of the measured star. +# DEC float The coordinates of the measured star. +# Exposure time float The exposure time of the DIMM observation. +# Telescope ID string source of the DIMM data + +dimm METADATA +# time DATETIME 2006-01-11T00:00:00 + sigmax F32 0.0 + sigmay F32 0.0 + fwhm F32 0.0 + ra F64 0.0 + decl F64 0.0 + expttime F32 0.0 + telescope_id STR 255 +END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/dome.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/dome.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/dome.md (revision 22160) @@ -0,0 +1,16 @@ +# Table 15: Dome Status Table +# Column Name Datatype Description +# Time date/time The time for which the dome status is valid. +# Azimuth float The azimuth of the dome. +# Open status boolean Whether the dome is open or not. +# Lights status boolean Whether lights are on in the dome or not. +# Track status boolean Whether dome is tracking telescope or not. + +dome METADATA +# time DATETIME 2006-01-11T00:00:00 + az F32 0.0 + open BOOL t + light BOOL t + # XXX is it possible for the dome slit to not track the telescope? ;) + track BOOL t +END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/fake.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/fake.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/fake.md (revision 22160) @@ -0,0 +1,29 @@ +fakeRun METADATA + fake_id S64 0 # Primary Key AUTO_INCREMENT + cam_id S64 0 # Key INDEX(fake_id, cam_id) fkey (cam_id) ref camRun(cam_id) + state STR 64 # Key + workdir STR 255 + label STR 64 # Key + reduction STR 64 # Reduction class + expgroup STR 64 # Key + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key + epoch UTC 0001-01-01T00:00:00Z +END + +fakeProcessedImfile METADATA + fake_id S64 0 # Primary Key fkey (fake_id) ref fakeRun(fake_id) + exp_id S64 64 # Primary Key fkey (exp_id, class_id) ref rawImfile(exp_id, class_id) + class_id STR 64 # Primary Key + uri STR 255 + dtime_fake F32 0.0 + hostname STR 64 + path_base STR 255 + fault S16 0 # Key NOT NULL + epoch UTC 0001-01-01T00:00:00Z +END + +fakeMask METADATA + label STR 64 # Primary Key +END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/flatcorr.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/flatcorr.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/flatcorr.md (revision 22160) @@ -0,0 +1,19 @@ + +# table of flat-field correction runs +flatcorrRun METADATA + corr_id S64 0 # Primary Key AUTO_INCREMENT + dvodb STR 64 + filter STR 64 + state STR 64 + workdir STR 255 + label STR 64 + stats STR 255 + region STR 255 +END + +# table of Exposure-level data used for each flat-field corrction run +flatcorrExp METADATA + corr_id S64 0 # Primary Key fkey(corr_id) ref flatcorrRun(corr_id) + chip_id S64 64 # Primary Key fkey(chip_id) ref chipRun(chip_id) +END + Index: /branches/eam_branches/eam_branch_20080806/dbconfig/guide.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/guide.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/guide.md (revision 22160) @@ -0,0 +1,5 @@ +guidePendingExp METADATA + guide_id S64 0 # Primary Key AUTO_INCREMENT + exp_id S64 64 # Key + recipe STR 64 +END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/ipp.m4 =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/ipp.m4 (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/ipp.m4 (revision 22160) @@ -0,0 +1,23 @@ +include(config.md) +dnl include(weather.md) +dnl include(skyp_transparency.md) +dnl include(skyp_absorption.md) +dnl include(skyp_emission.md) +dnl include(dimm.md) +dnl include(skyp_ir.md) +dnl include(dome.md) +dnl include(telescope.md) +dnl include(skycell.md) +include(tasks.md) +include(guide.md) +include(chip.md) +include(cam.md) +include(fake.md) +include(warp.md) +include(diff.md) +include(stack.md) +include(det.md) +include(magic.md) +include(calibration.md) +include(flatcorr.md) +include(pstamp.md) Index: /branches/eam_branches/eam_branch_20080806/dbconfig/magic.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/magic.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/magic.md (revision 22160) @@ -0,0 +1,47 @@ +# $Id: magic.md,v 1.10 2008-07-23 01:52:30 price Exp $ + +### Fault in magicRun indicates that the processing tree failed +magicRun METADATA + magic_id S64 0 # Primary Key AUTO_INCREMENT + exp_id S64 0 # Key + state STR 64 # Key + workdir STR 255 + workdir_state STR 255 # Key + label STR 64 # key + dvodb STR 255 + registered TAI NULL + fault S16 0 # Key +END + +magicInputSkyfile METADATA + magic_id S64 0 # Primary Key fkey(magic_id) ref magicRun(magic_id) + diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) + node STR 64 # +END + +magicTree METADATA + magic_id S64 0 # Key fkey(magic_id) ref magicRun(magic_id) + node STR 64 # Key INDEX(magic_id, node) + dep STR 64 # Key +END + +magicNodeResult METADATA + magic_id S64 0 # Primary Key fkey(magic_id) ref magicRun(magic_id) + node STR 64 # Primary Key fkey(magic_id, node) ref magicTree(magic_id, node) + uri STR 255 + fault S16 0 # Key +END + +magicMask METADATA + magic_id S64 0 # Primary Key fkey(magic_id) ref magicRun(magic_id) + uri STR 255 + streaks S32 0 + fault S16 0 # Key +END + +### I don't think we need this if magic produces mask descriptions. -- PAP. +#magicSkyfileMask METADATA +# magic_id S64 0 # Primary Key fkey(magic_id) ref magicRun(magic_id) +# diff_id S64 0 # Primary Key fkey(magic_id, diff_id) ref magicInputSkyfile(magic_id, diff_id) +# uri STR 255 # fkey(magic_id) ref magicMask(magic_id) +#END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/notes.txt =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/notes.txt (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/notes.txt (revision 22160) @@ -0,0 +1,41 @@ + +2007.06.08 : EAM + + * when changing the database schema: + * increment the pkg_version number on dbconfig/config.md + * increment the ippdb version number in ippTools/configure.ac (to match) + * increment the ippTools version number in ippTools/configure.ac + * build ippbd ('make src' in dbconfig) + * check in dbconfig, ippdb, and ippTools + +2007.06.04 : EAM + +I am adding additional fields which we will need for selecting the +input detrend images: + +rawExp : + user stat 1 + user stat 2 + user stat 3 + user stat 4 + user stat 5 + +detProcessedImfile, detProcessedExp: + fringe amplitude + user stat 1 + user stat 2 + user stat 3 + user stat 4 + user stat 5 + +detResidImfile, detResidExp: + binned stdev + fringe_0 + fringe_1 + fringe_2 + user stat 1 + user stat 2 + user stat 3 + user stat 4 + user stat 5 + Index: /branches/eam_branches/eam_branch_20080806/dbconfig/pstamp.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/pstamp.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/pstamp.md (revision 22160) @@ -0,0 +1,30 @@ +pstampDataStore METADATA + ds_id S64 0 # Primary Key AUTO_INCREMENT + state STR 64 + lastFileset STR 64 + outProduct STR 64 + uri STR 255 +END + +pstampRequest METADATA + req_id S64 0 # Primary Key AUTO_INCREMENT + ds_id S64 0 + state STR 64 + name STR 64 # UNIQUE + reqType STR 16 + outProduct STR 64 + uri STR 255 + fault S32 0 +END + +pstampJob METADATA + job_id S64 0 # Primary Key AUTO_INCREMENT + req_id S64 0 # Primary Key fkey(req_id) ref pstampRequest(req_id) + rownum STR 64 + state STR 64 + jobType STR 16 + fault S32 0 + uri STR 255 + outputBase STR 255 + args STR 511 +END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/skycell.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/skycell.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/skycell.md (revision 22160) @@ -0,0 +1,25 @@ +# $Id: skycell.md,v 1.4 2007-07-06 01:16:08 jhoblitt Exp $ + +skyCell METADATA + skycell_id STR 64 # Primary Key + tess_id STR 64 # Primary Key + ra1 F64 0.0 + decl1 F64 0.0 + ra2 F64 0.0 + decl2 F64 0.0 + ra3 F64 0.0 + decl3 F64 0.0 + ra4 F64 0.0 + decl4 F64 0.0 +END + +skyCellMap METADATA + skycell_id STR 64 # Primary Key + tess_id STR 64 # Primary Key + exp_id S64 64 # Primary Key + p3_version S32 0 # Primary Key +# class is not yet consistently carried through pXtools +# class STR 64 # Primary Key + class_id STR 64 # Primary Key +END + Index: /branches/eam_branches/eam_branch_20080806/dbconfig/skyp_absorption.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/skyp_absorption.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/skyp_absorption.md (revision 22160) @@ -0,0 +1,25 @@ +# Table 11: Skyprobe Line Absorption Table (sample entries) +# Column Name Datatype Description +# Time date/time The time the LRProbe observation was taken. +# Disperser ID string ID of the dispersing element +# Atm Component 1 float The strength of the 1st atmospheric component. +# Atm Component 2 float The strength of the 2nd atmospheric component. +# Atm Component 3 float The strength of the 3rd atmospheric component. +# Disperser ID string ID of the dispersing element +# Number of stars int Number of stars used to measure the absorptions. +# Astrometry coords The astrometry used on the LRProbe image. +# Exposure time float The exposure time of the LRProbe image. +# Sky brightness float The measured sky (surface) brightness, in physical units. + +skyp_absorption METADATA +# time DATETIME 2006-01-10T00:00:00 + disperser_id STR 255 + atmcomp1 F32 0.0 + atmcomp2 F32 0.0 + atmcomp3 F32 0.0 + nstars S32 0 + ra F64 0.0 + decl F64 0.0 + exptime F32 0.0 + sky_bright F64 0.0 +END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/skyp_emission.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/skyp_emission.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/skyp_emission.md (revision 22160) @@ -0,0 +1,20 @@ +# Table 12: Skyprobe Line Emission Table (sample entries) +# Column Name Datatype Description +# Time date/time The time the LRProbe observation was taken. +# Disperser ID string ID of the dispersing element +# Atm Component 1 float The strength of the 1st atmospheric component. +# Atm Component 2 float The strength of the 2nd atmospheric component. +# Atm Component 3 float The strength of the 3rd atmospheric component. +# Continuum float The strength of the continuum emission. +# Disperser ID string ID of the dispersing element +# Exposure time float The exposure time of the LRProbe image. + +skyp_emission METADATA +# time DATETIME 2006-01-11T00:00:00 + disperser_id STR 255 + atmcomp1 F32 0.0 + atmcomp2 F32 0.0 + atmcomp3 F32 0.0 + continuum F32 0.0 + exptime F32 0.0 +END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/skyp_ir.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/skyp_ir.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/skyp_ir.md (revision 22160) @@ -0,0 +1,18 @@ +# Table 14: Near IR Wide-field Camera Results Table +# Column Name Datatype Description +# Time date/time The time the NIR observation was taken. +# Sky brightness float The sky (surface) brightness in the NIR observation. +# Sky variance float The variance in the sky (surface) brightness. +# Astrometry coords The astrometry used on the NIR image. +# FOV X float field width +# FOV Y float field height + +skyp_ir METADATA +# time DATETIME 2006-01-11T00:00:00 + sky_bright F64 0.0 + sky_var F64 0.0 + ra F64 0.0 + decl F64 0.0 + fov_x F32 0.0 + fov_y F32 0.0 +END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/skyp_transparency.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/skyp_transparency.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/skyp_transparency.md (revision 22160) @@ -0,0 +1,21 @@ +# Table 10: SkyProbe Transparency Table (sample entries) +# Column Name Datatype Description +# Time date/time The time the SkyProbe image was taken. +# Filter string Filter used for SkyProbe image. +# Transparency float The derived transparency. +# Number of stars int The number of stars used to measure the transparency. +# Astrometry coords The astrometry used on the SkyProbe image. +# Exposure time float The exposure time of the SkyProbe image. +# Sky brightness float The measured sky (surface) brightness, counts / second + +skyp_transparency METADATA +# time DATETIME 2006-01-10T00:00:00 # Primary Key + filter STR 255 + trans F64 0.0 + nstars S32 0 +# astrom + ra F64 0.0 + decl F64 0.0 + exptime F32 0.0 + sky_bright F64 0.0 +END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/stack.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/stack.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/stack.md (revision 22160) @@ -0,0 +1,29 @@ +# $Id: stack.md,v 1.8 2008-07-16 19:20:46 price Exp $ + +stackRun METADATA + stack_id S64 0 # Primary Key AUTO_INCREMENT + state STR 64 # Key + workdir STR 255 + dvodb STR 255 + registered TAI NULL + skycell_id STR 64 # Key + tess_id STR 64 # Key + filter STR 64 +END + +stackInputSkyfile METADATA + stack_id S64 0 # Primary Key fkey(stack_id) ref stackRun(stack_id) + warp_id S64 0 # Primary Key fkey(warp_id) ref warpSkyfile(warp_id) +END + +stackSumSkyfile METADATA + stack_id S64 0 # Primary Key fkey(stack_id) ref stackRun(stack_id) + uri STR 255 + path_base STR 255 + bg F64 0.0 + bg_stdev F64 0.0 + dtime_stack F32 0.0 # Key + hostname STR 64 + good_frac F32 0.0 # Key + fault S16 0 # Key +END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/tasks.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/tasks.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/tasks.md (revision 22160) @@ -0,0 +1,226 @@ +# $Id: tasks.md,v 1.154 2008-05-31 02:19:17 jhoblitt Exp $ + +# this table records all exposure ID ever seen from the summit +# exp_name == fileset + +# note that dec is a MySQL reserved word +# note that use is a MySQL reserved word +# note that exp is a MySQL reserved word + +# for use with this stored procedure to generate exp_ids under Mysql 5+ +# +#CREATE FUNCTION genTag (exp_name varchar(64)) RETURNS VARCHAR(64) +#BEGIN +# UPDATE expTagCounter SET counter = LAST_INSERT_ID(counter + 1); +# RETURN CONCAT(exp_name, '.', LAST_INSERT_ID()); +#END +# +#expTagCounter METADATA +# counter U64 0 +#END + +pzDataStore METADATA + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + uri STR 255 + epoch UTC 0001-01-01T00:00:00Z +END + +# list of source exposures -- updated as exposures are seen +# summitExp.imfiles is updated as filesets are queried default value should be +# -1 as NULL or 0 might be a valid value (empty fileset) +summitExp METADATA + exp_name STR 64 # Primary Key + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + dateobs UTC NULL + exp_type STR 64 + uri STR 255 + imfiles S32 0 + fault S16 0 # Key NOT NULL + epoch UTC 0001-01-01T00:00:00Z +END + +# class == type of file +# class_id == type set id +# list of source images -- updated as exposures/filesets are queried +summitImfile METADATA + exp_name STR 64 # Primary Key fkey(exp_name, camera, telescope) ref summitExp(exp_name, camera, telescope) + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + file_id STR 64 # Key + bytes S32 0 + md5sum STR 32 + class STR 64 # Primary Key + class_id STR 64 # Primary Key + uri STR 255 + epoch UTC 0001-01-01T00:00:00Z +END + +# list of exposures that have had their imfiles/files registered (but not +# downloaded) +pzDownloadExp METADATA + exp_name STR 64 # Primary Key fkey(exp_name, camera, telescope) ref summitExp(exp_name, camera, telescope) + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + state STR 64 # Key + epoch UTC 0001-01-01T00:00:00Z +END + +pzDownloadImfile METADATA + exp_name STR 64 # Primary Key fkey(exp_name, camera, telescope) ref pzDownloadExp(exp_name, camera, telescope) + camera STR 64 # Primary Key fkey(exp_name, camera, telescope, class, class_id) ref summitImfile(exp_name, camera, telescope, class, class_id) + telescope STR 64 # Primary Key + class STR 64 # Primary Key + class_id STR 64 # Primary Key + uri STR 255 + fault S16 0 # Key NOT NULL + epoch UTC 0001-01-01T00:00:00Z +END + +newExp METADATA + exp_id S64 0 # Primary Key AUTO_INCREMENT + tmp_exp_name STR 64 # Key + tmp_camera STR 64 # Key + tmp_telescope STR 64 # Key + state STR 64 # Key + workdir STR 255 # destination for output files + workdir_state STR 64 # key + reduction STR 64 # Reduction class + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key + label STR 64 # Key + epoch UTC 0001-01-01T00:00:00Z +END + +# class needs to be carried here so it can go into rawImfile and be normalized +# from there +newImfile METADATA + exp_id S64 64 # Primary Key fkey(exp_id) ref newExp(exp_id) + tmp_class_id STR 64 # Primary Key + uri STR 255 + epoch UTC 0001-01-01T00:00:00Z +END + +# paired with rawImfile +rawExp METADATA + exp_id S64 64 # Primary Key fkey(exp_id) ref newExp(exp_id) + exp_name STR 64 # Key + camera STR 64 + telescope STR 64 + dateobs UTC 0001-01-01T00:00:00Z + exp_tag STR 255 + exp_type STR 64 + filelevel STR 64 + workdir STR 255 # destination for output files + reduction STR 64 # Reduction class + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key + filter STR 64 + comment STR 80 + airmass F32 0.0 + ra F64 0.0 + decl F64 0.0 + exp_time F32 0.0 + sat_pixel_frac F32 0.0 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + alt F64 0.0 + az F64 0.0 + ccd_temp F32 0.0 + posang F64 0.0 + m1_x F32 0.0 + m1_y F32 0.0 + m1_z F32 0.0 + m1_tip F32 0.0 + m1_tilt F32 0.0 + m2_x F32 0.0 + m2_y F32 0.0 + m2_z F32 0.0 + m2_tip F32 0.0 + m2_tilt F32 0.0 + env_temperature F32 0.0 + env_humidity F32 0.0 + env_wind_speed F32 0.0 + env_wind_dir F32 0.0 + teltemp_m1 F32 0.0 + teltemp_m1cell F32 0.0 + teltemp_m2 F32 0.0 + teltemp_spider F32 0.0 + teltemp_truss F32 0.0 + teltemp_extra F32 0.0 + pon_time F32 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + object STR 64 + solang F32 0.0 + hostname STR 64 + fault S16 0 # Key NOT NULL + epoch UTC 0001-01-01T00:00:00Z +END + +rawImfile METADATA + exp_id S64 64 # Primary Key fkey(exp_id, tmp_class_id) ref newImfile(exp_id, tmp_class_id) + exp_name STR 64 # UINDEX(exp_id, tmp_class_id) + camera STR 64 + telescope STR 64 + dateobs UTC 0001-01-01T00:00:00Z + tmp_class_id STR 64 # Key + class_id STR 64 # Primary Key + uri STR 255 + exp_type STR 64 +# This field is used to set the per exp filelevel. Thus the values for all of +# the imfiles in an exposure need to be sanity checked to make sure that this +# value is in argeement. + filelevel STR 64 + filter STR 64 + comment STR 80 + airmass F32 0.0 + ra F64 0.0 + decl F64 0.0 + exp_time F32 0.0 + sat_pixel_frac F32 0.0 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + alt F64 0.0 + az F64 0.0 + ccd_temp F32 0.0 + posang F64 0.0 + m1_x F32 0.0 + m1_y F32 0.0 + m1_z F32 0.0 + m1_tip F32 0.0 + m1_tilt F32 0.0 + m2_x F32 0.0 + m2_y F32 0.0 + m2_z F32 0.0 + m2_tip F32 0.0 + m2_tilt F32 0.0 + env_temperature F32 0.0 + env_humidity F32 0.0 + env_wind_speed F32 0.0 + env_wind_dir F32 0.0 + teltemp_m1 F32 0.0 + teltemp_m1cell F32 0.0 + teltemp_m2 F32 0.0 + teltemp_spider F32 0.0 + teltemp_truss F32 0.0 + teltemp_extra F32 0.0 + pon_time F32 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + object STR 64 + hostname STR 64 + fault S16 0 # Key NOT NULL + epoch UTC 0001-01-01T00:00:00Z +END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/telescope.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/telescope.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/telescope.md (revision 22160) @@ -0,0 +1,18 @@ +# Table 16: Telescope Status +# Column Name Datatype Description +# Time date/time The time for which the telescope status is valid. +# Guide status enum The status of the guiding. +# Altitude float The telescope altitude. +# Azimuth float The telescope azimuth. +# RA float The telescope Right Ascension (ICRS ~ J2000). +# Dec float The telescope Declination (ICRS ~ J2000). + +telescope METADATA +# time DATETIME 2006-01-11T00:00:00 # Primary Key +# XXX there is currently no way to declare an enum - use str or int instead? + guide STR 255 + alt F32 0.0 + az F32 0.0 + ra F64 0.0 + decl F64 0.0 +END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/warp.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/warp.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/warp.md (revision 22160) @@ -0,0 +1,56 @@ +# $Id: warp.md,v 1.17 2008-06-09 22:55:57 jhoblitt Exp $ + +# +# We have at least 3 different run types +# a) single epoch differing +# b) magic run operates on "complete" exposures +# c) multiple epoch differing/stacking operations +# + +# define a new warprun for a single skycell +warpRun METADATA + warp_id S64 0 # Primary Key AUTO_INCREMENT + fake_id S64 0 # Key INDEX(warp_id, fake_id) fkey(fake_id) ref camProcessedExp(fake_id) + mode STR 64 # Key + state STR 64 # Key + workdir STR 255 + workdir_state STR 64 # Key + label STR 64 # key + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key + registered TAI NULL +# if magic is T then look for the exp_id in the magic output tables + magiced BOOL f # Key +END + +warpSkyCellMap METADATA + warp_id S64 0 # Primary Key fkey(warp_id) ref warpRun(warp_id) + skycell_id STR 64 # Primary Key + tess_id STR 64 # Primary Key + class_id STR 64 # Primary Key + fault S16 0 # Key +END + +warpSkyfile METADATA + warp_id S64 0 # Primary Key fkey(warp_id, skycell_id, tess_id) ref warpSkyCellMap(warp_id, skycell_id, tess_id) + skycell_id STR 64 # Primary Key + tess_id STR 64 # Primary Key + uri STR 255 + path_base STR 255 + bg F64 0.0 + bg_stdev F64 0.0 + dtime_warp F32 0.0 + hostname STR 64 + good_frac F32 0.0 # Key + xmin S32 0 + xmax S32 0 + ymin S32 0 + ymax S32 0 + ignored BOOL f # Key + fault S16 0 # Key +END + +warpMask METADATA + label STR 64 # Primary Key +END Index: /branches/eam_branches/eam_branch_20080806/dbconfig/weather.md =================================================================== --- /branches/eam_branches/eam_branch_20080806/dbconfig/weather.md (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/dbconfig/weather.md (revision 22160) @@ -0,0 +1,19 @@ +# Table 9: Weather Table: some sample weather points +# Column Name Datatype Description +# Time date/time The time the weather information was measured. +# Temperature 01 float The external temperature +# Temperature 02 float The temperature at top of the dome +# Temperature 03 float The temperature on the primary mirror +# Humidity float The relative humidity. +# Pressure float The (external) atmospheric pressure. + +weather METADATA +# time DATETIME 2006-01-10T00:00:00 # Primary Key + temp01 F32 0.0 + humi01 F32 0.0 + temp02 F32 0.0 + humi02 F32 0.0 + temp03 F32 0.0 + humi03 F32 0.0 + pressure F32 0.0 +END Index: /branches/eam_branches/eam_branch_20080806/ippTasks/.cvsignore =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/.cvsignore (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/.cvsignore (revision 22160) @@ -0,0 +1,9 @@ +Makefile +Makefile.in +aclocal.m4 +autom4te.cache +config.log +config.status +configure +install-sh +missing Index: /branches/eam_branches/eam_branch_20080806/ippTasks/Makefile.am =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/Makefile.am (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/Makefile.am (revision 22160) @@ -0,0 +1,42 @@ +task_files = \ + detrend.mkruns.pro \ + detrend.norm.pro \ + detrend.process.pro \ + detrend.reject.pro \ + detrend.resid.pro \ + detrend.stack.pro \ + automate.pro \ + pantasks.pro \ + register.pro \ + chip.pro \ + camera.pro \ + fake.pro \ + warp.pro \ + magic.pro \ + diff.pro \ + stack.pro \ + summit.copy.pro \ + replicate.pro \ + pstamp.pro + +other_files = \ + chiphosts.manoa.config \ + chiphosts.mhpcc.config \ + simtest.basic.config \ + site.manoa.pro \ + site.mhpcc.pro \ + simtest.pro \ + simtest.auto \ + simtest.det \ + isp.det + +pantasksdir = $(datadir)/pantasks +ipptasksdir = $(pantasksdir)/modules +ipptasks_DATA = $(task_files) $(other_files) + +install-data-hook: + chmod 0755 $(panstasks) $(ipptasksdir) + +EXTRA_DIST = $(task_files) $(other_files) + +ACLOCAL_AMFLAGS = -I m4 Index: /branches/eam_branches/eam_branch_20080806/ippTasks/autogen.sh =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/autogen.sh (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/autogen.sh (revision 22160) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=ippTasks +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +#($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $LIBTOOlIZE installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" +# DIE=1 +#} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +#($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $AUTOHEADER installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" +# DIE=1 +#} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +#$LIBTOOLIZE --copy --force || echo "$LIBTOOlIZE failed" +$ACLOCAL || echo "$ACLOCAL failed" +#$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: /branches/eam_branches/eam_branch_20080806/ippTasks/automate.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/automate.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/automate.pro (revision 22160) @@ -0,0 +1,432 @@ +## automate.pro : autorun the detrend analysis : -*- sh -*- + +# example automation stage +# automate METADATA +# name STR DARK +# check STR "detselect -search -inst SIMTEST -det_type BIAS -dbname eamtest" +# ncheck S32 1 +# launch STR "dettool -definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type DARK -select_exp_type DARK -dbname eamtest" +# block STR "dettool -runs -active -det_type DARK -dbname eamtest" +# END + +# an entry in the 'automate' book goes through the following states: +# INIT (book loaded from input file) +# CHECK (stage prereqs being checked) +# READY (prereqs satisfield, ready to be launched) +# LAUNCH (action being performed) +# DONE (action finished) + +macro automate.reset + ## probably should not always init + book init automate +end + +macro automate.load + if ($0 != 3) + echo "USAGE: automate.load (filename) (dbname)" + break + end + queueload tmp -x "cat $MODULES:0/$1" + + pwd -var cwd + + ## interpolate standard values + queuesubstr tmp @DBNAME@ $2 + queuesubstr tmp @CWD@ $cwd + + ipptool2book tmp automate -key name -uniq -setword pantaskState INIT.BLOCK + queuedelete tmp + + ## Pull out the ones that are to be run regularly + local npages + book npages automate -var npages + for i 0 $npages + book getpage automate $i -var pageName + if ("$pageName" != "NULL") + book getword automate $pageName regular -var regularCommand + if ("$regularCommand" != "NULL") + book setword automate $pageName pantaskState INIT.REGULAR + end + end + end + +end + +macro automate.on + task automate.block + active true + end + task automate.check + active true + end + task automate.launch + active true + end + task automate.regular + active true + end +end + +macro automate.off + task automate.block + active false + end + task automate.check + active false + end + task automate.launch + active false + end + task automate.regular + active false + end +end + +$automate_Nblock = -1 +task automate.block + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + task.exec + local Npage pageName + + # how many pages are waiting to be started? + book npages automate -var Npage -key pantaskState INIT.BLOCK + if ($Npage == 0) + if ($VERBOSE >= 2) + echo "no entries in INIT.BLOCK state" + end + break + end + + # cycle over the number of INIT.BLOCK pages: no point to loop over the others + $automate_Nblock ++ + if ($automate_Nblock >= $Npage) set automate_Nblock = 0 + + # search the automate book for an entry which is unstarted (state INIT.BLOCK) + book getpage automate $automate_Nblock -var pageName -key pantaskState INIT.BLOCK + if ("$pageName" == "NULL") + if ($VERBOSE >= 2) + echo "entry $automate_Nblock not in INIT.BLOCK state" + end + break + end + + book getword automate $pageName block -var blockCommand + if (("$blockCommand" == "NULL") || ("$blockCommand" == "NONE")) + # if there is no block needed, we can immediate progress to the next stage (INIT.CHECK) + book setword automate $pageName pantaskState INIT.CHECK + if ($VERBOSE >= 2) + echo "$pageName is ready : INIT.CHECK" + end + break + end + + book setword automate $pageName pantaskState RUN.BLOCK + + if ($VERBOSE >= 2) + echo "starting automate block for $pageName" + echo "command $blockCommand" + end + + options $pageName + command $blockCommand + end + + task.exit $EXIT_SUCCESS + local pageName Npage + + $pageName = $options:0 + + # convert 'stdout' to book format + # XXX to use other tests, we'll need to modify this + ipptool2book stdout tmpBlock + if ($VERBOSE > 2) + book listbook tmpBlock + end + + # if the block test returns any valid pages (valid results), the block is set, don't move to check + book npages tmpBlock -var Npage + if ($Npage == 0) + if ($VERBOSE >= 2) + echo "$pageName is not blocked, ready for CHECK" + end + book setword automate $pageName pantaskState INIT.CHECK + else + if ($VERBOSE >= 2) + echo "$pageName is blocked, not ready for CHECK" + end + book setword automate $pageName pantaskState INIT.BLOCK + end + + # drop the detExt book after we've grabbed the state + book delete tmpBlock + end + + # all failures here (what state?) + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +$automate_Ncheck = -1 +task automate.check + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + task.exec + local Npage pageName + + # how many pages are waiting to be started? + book npages automate -var Npage -key pantaskState INIT.CHECK + if ($Npage == 0) + if ($VERBOSE >= 2) + echo "no entries in INIT.CHECK state" + end + break + end + + # cycle over the number of INIT.CHECK pages: no point to loop over the others + $automate_Ncheck ++ + if ($automate_Ncheck >= $Npage) set automate_Ncheck = 0 + + # search the automate book for an entry which is unstarted (state INIT.CHECK) + book getpage automate $automate_Ncheck -var pageName -key pantaskState INIT.CHECK + if ("$pageName" == "NULL") + if ($VERBOSE >= 2) + echo "entry $automate_Ncheck not in INIT.CHECK state" + end + break + end + + book getword automate $pageName check -var checkCommand + if (("$checkCommand" == "NULL") || ("$checkCommand" == "NONE")) + # if there is no check needed, we can immediate progress to the next stage (INIT.LAUNCH) + book setword automate $pageName pantaskState INIT.LAUNCH + if ($VERBOSE >= 2) + echo "$pageName is ready : INIT.LAUNCH" + end + break + end + + book setword automate $pageName pantaskState RUN.CHECK + + if ($VERBOSE >= 2) + echo "starting automate check for $pageName" + echo "command $checkCommand" + end + + options $pageName + command $checkCommand + end + + task.exit $EXIT_SUCCESS + local pageName Npage + + $pageName = $options:0 + + # convert 'stdout' to book format + # XXX to use other tests, we'll need to modify this + ipptool2book stdout tmpCheck + if ($VERBOSE > 2) + book listbook tmpCheck + end + + book getword automate $pageName ncheck -var Ncheck + if ("$Ncheck" == "NULL") + $Ncheck = 1 + end + book npages tmpCheck -var Npage + if ($Npage < $Ncheck) + if ($VERBOSE >= 2) + echo "$pageName not ready for LAUNCH" + end + book setword automate $pageName pantaskState INIT.CHECK + else + if ($VERBOSE >= 2) + echo "$pageName is ready for LAUNCH" + end + book setword automate $pageName pantaskState INIT.LAUNCH + end + + # drop the detExt book after we've grabbed the state + book delete tmpCheck + end + + # all failures here (what state?) + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +$automate_Nlaunch = 0 +task automate.launch + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + task.exec + local Npage pageName + + # how many pages are waiting to be started? + book npages automate -var Npage -key pantaskState INIT.LAUNCH + if ($Npage == 0) + if ($VERBOSE >= 2) + echo "no entries in INIT.LAUNCH state" + end + break + end + + # cycle over the number of INIT.LAUNCH pages: no point to loop over the others + $automate_Nlaunch ++ + if ($automate_Nlaunch >= $Npage) set automate_Nlaunch = 0 + + # search the automate book for an entry which is unstarted (state INIT.LAUNCH) + book getpage automate $automate_Nlaunch -var pageName -key pantaskState INIT.LAUNCH + if ("$pageName" == "NULL") + if ($VERBOSE >= 2) + echo "entry $automate_Nlaunch not in INIT.LAUNCH state" + end + break + end + + book getword automate $pageName launch -var launchCommand + if (("$launchCommand" == "NULL") || ("$launchCommand" == "NONE")) + # if there is no launch needed, we can immediate progress to the next stage (INIT.LAUNCH) + book setword automate $pageName pantaskState INIT.LAUNCH + if ($VERBOSE >= 2) + echo "$pageName is ready : INIT.LAUNCH" + end + break + end + + book setword automate $pageName pantaskState RUN.LAUNCH + + if ($VERBOSE >= 2) + echo "starting automate launch for $pageName" + echo "command $launchCommand" + end + + options $pageName + command $launchCommand + end + + task.exit $EXIT_SUCCESS + local pageName Npage + + $pageName = $options:0 + + book setword automate $pageName pantaskState DONE.LAUNCH + end + + # all failures here (what state?) + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +$automate_Nregular = 0 +task automate.regular + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + task.exec + local Npage pageName + + # how many pages are waiting to be started? + book npages automate -var Npage -key pantaskState INIT.REGULAR + if ($Npage == 0) + if ($VERBOSE >= 2) + echo "no entries in INIT.REGULAR state" + end + break + end + + # cycle over the number of INIT.REGULAR pages: no point to loop over the others + $automate_Nregular ++ + if ($automate_Nregular >= $Npage) set automate_Nregular = 0 + + # search the automate book for an entry which is unstarted (state INIT.REGULAR) + book getpage automate $automate_Nregular -var pageName -key pantaskState INIT.REGULAR + if ("$pageName" == "NULL") + if ($VERBOSE >= 2) + echo "entry $automate_Nregular not in INIT.REGULAR state" + end + break + end + + book getword automate $pageName regular -var regularCommand + if (("$regularCommand" == "NULL") || ("$regularCommand" == "NONE")) + if ($VERBOSE >= 2) + echo "Warning: $pageName has no regular command" + end + break + end + + book setword automate $pageName pantaskState RUN.REGULAR + + if ($VERBOSE >= 2) + echo "starting automate regular for $pageName" + echo "command $regularCommand" + end + + options $pageName + command $regularCommand + end + + task.exit $EXIT_SUCCESS + local pageName Npage + + $pageName = $options:0 + + book setword automate $pageName pantaskState INIT.REGULAR + end + + # all failures here (what state?) + task.exit default + book setword automate $pageName pantaskState FAIL.REGULAR + showcommand failure + end + + # operation times out? + task.exit timeout + book setword automate $pageName pantaskState INIT.REGULAR + showcommand timeout + end +end + Index: /branches/eam_branches/eam_branch_20080806/ippTasks/calibration.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/calibration.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/calibration.pro (revision 22160) @@ -0,0 +1,173 @@ +## calibration.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the calibration stage +## these tasks use the book 'calBook' + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/calibration +exec mkdir -p $LOGSUBDIR + +book init calBook + +macro calibration.reset + book init calBook +end + +macro calibration.status + book listbook calBook +end + +macro calibration.on + task calibration.init + active true + end +end + +macro calibration.off + task calibration.init + active false + end +end + +# these variables will cycle through the known ippdb database names +$calInit_DB = 0 + +# we have three steps here: +# 1) get the list of dvo databases: caltool -dbs +# 2) define a new calibration run for each of the dvo dbs +# caltool + +# create new calibration entries for the currently known DVO databases +# run this multiple times once an hour - on pass for each db +task calibration.load + host local + + # check the list of available dvo dbs regularly + periods -poll 10 + periods -exec 900 + periods -timeout 60 + + # this is a strange construction; what is the purpose? + if ($DB:n == 0) + npending $DB:n + else + npending 1 + end + + # define the command (does not depend on previous queries) + if ($DB:n == 0) + command caltool -dbs -active true + else + # save the DB name for the exit tasks + # note that this DB name refers to the ippdb, not the dvodb + option $DB:$calInit_DB + command caltool -dbs -dbname $DB:$calInit_DB + $calInit_DB ++ + if ($calInit_DB >= $DB:n) set calInit_DB = 0 + end + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/calibration.log + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + # XXX have ippTools report the dbname? + ipptool2book stdout calBook -key dbname:cal_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook calBook + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup calBook + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# create new calibration entries for the currently known DVO databases +# run this multiple times once an hour - one pass for each db +# can we include information that the db has been updated without a recent cal analysis? +task calibration.resort + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 60 + # trange Hourly@00:00 Hourly@00:10 + # every hour on the hour + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/calibration.log + + task.exec + book npages calBook -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in calBook + # the sequencing in this task set is by the pantasksState (see notes.txt) + book getpage calBook 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # get the current ST: + $RAs = 15*($ST - 1) + $RAe = 15*($ST - 2) + $DECs = -90.0 + $DECe = +90.0 + + $REGION = "$RAs,$RAe:$DECs,$DECe" + + book setword calBook $pageName pantaskState RUN + book setword calBook $pageName region $REGION + + # XXX probably need to set the output / log based on WORKDIR... + book getword calBook $pageName cal_id -var ID + book getword calBook $pageName dvodb -var DVODB + book getword calBook $pageName dbname -var DBNAME + + # specify choice of remote host + # set a specific DVO host here + if ($PARALLEL) + host anyhost + else + host local + end + + # save the pageName for future reference below + options $pageName + + # XXX do we modify relphot to loop over the filters? + $run = calibrate_dvo.pl --cal_id $ID --dvodb $DVODB --region $REGION + add_standard_args run + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit calBook $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword calBook $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080806/ippTasks/camera.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/camera.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/camera.pro (revision 22160) @@ -0,0 +1,287 @@ +## camera.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the camera analysis stage +## these tasks use the book camPendingExp + +# test for required global variables +check.globals + +book init camPendingExp +book init camPendingCleanup + +macro camera.status + book listbook camPendingExp + book listbook camPendingCleanup +end + +macro camera.reset + book init camPendingExp + book init camPendingCleanup +end + +macro camera.on + task camera.exp.load + active true + end + task camera.exp.run + active true + end + task camera.cleanup.load + active false + end + task camera.cleanup.run + active false + end +end + +macro camera.off + task camera.exp.load + active false + end + task camera.exp.run + active false + end + task camera.cleanup.load + active false + end + task camera.cleanup.run + active false + end +end + +# this variable will cycle through the known database names +$camera_DB = 0 + +# select images ready for camera analysis +# new entries are added to camPendingExp +# skip already-present entries +task camera.exp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/camera.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command camtool -pendingexp -limit 20 + # echo command camtool -pendingexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$camera_DB + command camtool -pendingexp -limit 20 -dbname $DB:$camera_DB + # echo command camtool -pendingexp -limit 20 -dbname $DB:$camera_DB + $camera_DB ++ + if ($camera_DB >= $DB:n) set camera_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout camPendingExp -key cam_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook camPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup camPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the cameraexposure script on pending images +task camera.exp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10 + + task.exec + book npages camPendingExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in camPendingExp (pantaskState == INIT) + book getpage camPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword camPendingExp $pageName pantaskState RUN + book getword camPendingExp $pageName camera -var CAMERA + book getword camPendingExp $pageName exp_tag -var EXP_TAG + book getword camPendingExp $pageName cam_id -var CAM_ID + book getword camPendingExp $pageName workdir -var WORKDIR_TEMPLATE + book getword camPendingExp $pageName dvodb -var DVODB + book getword camPendingExp $pageName dbname -var DBNAME + book getword camPendingExp $pageName reduction -var REDUCTION + + # specify choice of remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s/%s.cm.%s" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID + + stdout $LOGDIR/camera.exp.log + stderr $LOGDIR/camera.exp.log + + $run = camera_exp.pl --exp_tag $EXP_TAG --cam_id $CAM_ID --camera $CAMERA --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + if ("$DVODB" != "NULL") + $run = $run --dvodb $DVODB + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit camPendingExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword camPendingExp $options:0 pantaskState TIMEOUT + end +end + +# this variable will cycle through the known database names +$camera_cleanup_DB = 0 + +# select images ready for cam analysis +# new entries are added to camPendingImfile +# skip already-present entries +task camera.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/camera.cleanup.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command camtool -pendingcleanuprun -limit 20 + else + # save the DB name for the exit tasks + option $DB:$camera_cleanup_DB + command camtool -pendingcleanuprun -limit 20 -dbname $DB:$camera_cleanup_DB + $camera_cleanup_DB ++ + if ($camera_cleanup_DB >= $DB:n) set camera_cleanup_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout camPendingCleanup -key cam_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook camPendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup camPendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task camera.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages camPendingCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in camPendingCleanup (pantaskState == INIT) + book getpage camPendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword camPendingCleanup $pageName pantaskState RUN + book getword camPendingCleanup $pageName camera -var CAMERA + book getword camPendingCleanup $pageName state -var CLEANUP_MODE + book getword camPendingCleanup $pageName cam_id -var CAM_ID + book getword camPendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and cam (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/camera.cleanup.log + stderr $LOGDIR/camera.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage camera --stage_id $CAM_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit camPendingCleanup $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword camPendingCleanup $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080806/ippTasks/chip.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/chip.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/chip.pro (revision 22160) @@ -0,0 +1,287 @@ +## chip.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the phase 0 stage +## these tasks use the book chipPendingImfile + +## XXX the cleanup stage probably needs to mimic the chip processing stages, with an exposure and imfile layer +## can we define the cleanup stages in a separate file derived from this file? + +# test for required global variables +check.globals + +book init chipPendingImfile +book init chipPendingCleanup + +macro chip.status + book listbook chipPendingImfile + book listbook chipPendingCleanup +end + +macro chip.reset + book init chipPendingImfile + book init chipPendingCleanup +end + +macro chip.on + task chip.imfile.load + active true + end + task chip.imfile.run + active true + end + task chip.cleanup.load + active true + end + task chip.cleanup.run + active true + end +end + +macro chip.off + task chip.imfile.load + active false + end + task chip.imfile.run + active false + end + task chip.cleanup.load + active false + end + task chip.cleanup.run + active false + end +end + +# this variable will cycle through the known database names +$chip_DB = 0 + +# select images ready for chip analysis +# new entries are added to chipPendingImfile +# skip already-present entries +task chip.imfile.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/chip.imfile.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command chiptool -pendingimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$chip_DB + command chiptool -pendingimfile -limit 20 -dbname $DB:$chip_DB + $chip_DB ++ + if ($chip_DB >= $DB:n) set chip_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout chipPendingImfile -key chip_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook chipPendingImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup chipPendingImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the chip_imfile.pl script on pending images +task chip.imfile.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages chipPendingImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in chipPendingImfile (pantaskState == INIT) + book getpage chipPendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword chipPendingImfile $pageName pantaskState RUN + book getword chipPendingImfile $pageName camera -var CAMERA + book getword chipPendingImfile $pageName exp_id -var EXP_ID + book getword chipPendingImfile $pageName exp_tag -var EXP_TAG + book getword chipPendingImfile $pageName chip_id -var CHIP_ID + book getword chipPendingImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword chipPendingImfile $pageName class_id -var CLASS_ID + book getword chipPendingImfile $pageName uri -var URI + book getword chipPendingImfile $pageName dbname -var DBNAME + book getword chipPendingImfile $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s/%s.ch.%s" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_ID + + stdout $LOGDIR/chip.imfile.log + stderr $LOGDIR/chip.imfile.log + + $run = chip_imfile.pl --exp_id $EXP_ID --chip_id $CHIP_ID --class_id $CLASS_ID --uri $URI --camera $CAMERA --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit chipPendingImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword chipPendingImfile $options:0 pantaskState TIMEOUT + end +end + +# this variable will cycle through the known database names +$chip_cleanup_DB = 0 + +# select images ready for chip analysis +# new entries are added to chipPendingImfile +# skip already-present entries +task chip.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/chip.cleanup.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command chiptool -pendingcleanuprun -limit 20 + else + # save the DB name for the exit tasks + option $DB:$chip_cleanup_DB + command chiptool -pendingcleanuprun -limit 20 -dbname $DB:$chip_cleanup_DB + $chip_cleanup_DB ++ + if ($chip_cleanup_DB >= $DB:n) set chip_cleanup_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout chipPendingCleanup -key chip_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook chipPendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup chipPendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task chip.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages chipPendingCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in chipPendingCleanup (pantaskState == INIT) + book getpage chipPendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword chipPendingCleanup $pageName pantaskState RUN + book getword chipPendingCleanup $pageName camera -var CAMERA + book getword chipPendingCleanup $pageName state -var CLEANUP_MODE + book getword chipPendingCleanup $pageName chip_id -var CHIP_ID + book getword chipPendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/chip.cleanup.log + stderr $LOGDIR/chip.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage chip --stage_id $CHIP_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit chipPendingCleanup $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword chipPendingCleanup $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080806/ippTasks/chiphosts.manoa.config =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/chiphosts.manoa.config (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/chiphosts.manoa.config (revision 22160) @@ -0,0 +1,19 @@ +## basic chiphost table: use the defaults for everything + +chiphosts MULTI + +chiphosts METADATA + camera STR MEGACAM +END + +chiphosts METADATA + camera STR CFH12K +END + +chiphosts METADATA + camera STR GPC1 +END + +chiphosts METADATA + camera STR gpc1 +END Index: /branches/eam_branches/eam_branch_20080806/ippTasks/chiphosts.mhpcc.config =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/chiphosts.mhpcc.config (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/chiphosts.mhpcc.config (revision 22160) @@ -0,0 +1,170 @@ +chiphosts MULTI + +chiphosts METADATA + camera STR MEGACAM + ccd00 STR po00 + ccd01 STR po01 + ccd02 STR po02 + ccd03 STR po03 + ccd04 STR po04 + ccd05 STR po05 + ccd06 STR po06 + ccd07 STR po07 + ccd08 STR po08 + ccd09 STR po09 +END + +chiphosts METADATA + camera STR CFH12K + ccd00 STR po10 + ccd01 STR po11 + ccd02 STR po12 +END + +chiphosts METADATA + camera STR GPC1 + + XY01 STR ipp005 + XY02 STR ipp005 + XY03 STR ipp005 + XY04 STR ipp005 + XY05 STR ipp005 + XY06 STR ipp005 + + XY10 STR ipp006 + XY11 STR ipp006 + XY12 STR ipp006 + XY13 STR ipp006 + XY14 STR ipp006 + XY15 STR ipp006 + + XY16 STR ipp007 + XY17 STR ipp007 + XY20 STR ipp007 + XY21 STR ipp007 + XY22 STR ipp007 + XY23 STR ipp007 + + XY24 STR ipp006 + XY25 STR ipp009 + XY26 STR ipp010 + XY27 STR ipp011 + XY30 STR ipp016 + XY31 STR ipp020 + + XY32 STR ipp009 + XY33 STR ipp009 + XY34 STR ipp009 + XY35 STR ipp009 + XY36 STR ipp009 + XY37 STR ipp009 + + XY40 STR ipp010 + XY41 STR ipp010 + XY42 STR ipp010 + XY43 STR ipp010 + XY44 STR ipp010 + XY45 STR ipp010 + + XY46 STR ipp011 + XY47 STR ipp011 + XY50 STR ipp011 + XY51 STR ipp011 + XY52 STR ipp011 + XY53 STR ipp011 + + XY54 STR ipp016 + XY55 STR ipp016 + XY56 STR ipp016 + XY57 STR ipp016 + XY60 STR ipp016 + XY61 STR ipp016 + + XY62 STR ipp020 + XY63 STR ipp020 + XY64 STR ipp020 + XY65 STR ipp020 + XY66 STR ipp020 + XY67 STR ipp020 + + XY71 STR ipp021 + XY72 STR ipp021 + XY73 STR ipp021 + XY74 STR ipp021 + XY75 STR ipp021 + XY76 STR ipp021 +END + +chiphosts METADATA + camera STR gpc1 + + ota01 STR ipp005 + ota02 STR ipp005 + ota03 STR ipp005 + ota04 STR ipp005 + ota05 STR ipp005 + ota06 STR ipp005 + + ota10 STR ipp006 + ota11 STR ipp006 + ota12 STR ipp006 + ota13 STR ipp006 + ota14 STR ipp006 + ota15 STR ipp006 + + ota16 STR ipp007 + ota17 STR ipp007 + ota20 STR ipp007 + ota21 STR ipp007 + ota22 STR ipp007 + ota23 STR ipp007 + + ota24 STR ipp006 + ota25 STR ipp009 + ota26 STR ipp010 + ota27 STR ipp011 + ota30 STR ipp016 + ota31 STR ipp020 + + ota32 STR ipp009 + ota33 STR ipp009 + ota34 STR ipp009 + ota35 STR ipp009 + ota36 STR ipp009 + ota37 STR ipp009 + + ota40 STR ipp010 + ota41 STR ipp010 + ota42 STR ipp010 + ota43 STR ipp010 + ota44 STR ipp010 + ota45 STR ipp010 + + ota46 STR ipp011 + ota47 STR ipp011 + ota50 STR ipp011 + ota51 STR ipp011 + ota52 STR ipp011 + ota53 STR ipp011 + + ota54 STR ipp016 + ota55 STR ipp016 + ota56 STR ipp016 + ota57 STR ipp016 + ota60 STR ipp016 + ota61 STR ipp016 + + ota62 STR ipp020 + ota63 STR ipp020 + ota64 STR ipp020 + ota65 STR ipp020 + ota66 STR ipp020 + ota67 STR ipp020 + + ota71 STR ipp021 + ota72 STR ipp021 + ota73 STR ipp021 + ota74 STR ipp021 + ota75 STR ipp021 + ota76 STR ipp021 +END Index: /branches/eam_branches/eam_branch_20080806/ippTasks/configure.ac =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/configure.ac (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/configure.ac (revision 22160) @@ -0,0 +1,14 @@ +AC_PREREQ(2.59) + +AC_INIT([ippTasks], [1.1.0], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([pantasks.pro]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_MAINTAINER_MODE + +AC_PROG_INSTALL + +AC_CONFIG_FILES([ + Makefile +]) +AC_OUTPUT Index: /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.correct.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.correct.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.correct.pro (revision 22160) @@ -0,0 +1,294 @@ +## detrend.correct.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend correction stage +## these tasks use the books detPendingCorrectedImfile and detPendingCorrectedExp + +# test for required global variables +check.globals + +book init detPendingCorrectedImfile +book init detPendingCorrectedExp + +macro detcorr.reset + book init detPendingCorrectedImfile + book init detPendingCorrectedExp +end + +macro detcorr.status + echo detPendingCorrectedImfile + book listbook detPendingCorrectedImfile + echo detPendingCorrectedExp + book listbook detPendingCorrectedExp +end + +macro detcorr.on + task detrend.correct.load + active true + end + task detrend.correct.run + active true + end + task detrend.correctexp.load + active true + end + task detrend.correctexp.run + active true + end +end + +macro detcorr.off + task detrend.correct.load + active false + end + task detrend.correct.run + active false + end + task detrend.correctexp.load + active false + end + task detrend.correctexp.run + active false + end +end + + +# these variables will cycle through the known database names +$detPendingCorrectedImfile_DB = 0 +$detPendingCorrectedExp_DB = 0 + +# select images ready for copy +# new entries are added to detPendingCorrectedImfile +# compare the new list with the ones already selected +task detrend.correct.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detcorr.load.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tocorrectedimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingCorrectedImfile_DB + command dettool -tocorrectedimfile -limit 20 -dbname $DB:$detPendingCorrectedImfile_DB + $detPendingCorrectedImfile_DB ++ + if ($detPendingCorrectedImfile_DB >= $DB:n) set detPendingCorrectedImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingCorrectedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingCorrectedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingCorrectedImfile + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand + end +end + +# run detrend_correct_imfile.pl on pending images +task detrend.correct.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingCorrectedImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingCorrectedImfile + book getpage detPendingCorrectedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingCorrectedImfile $pageName pantaskState RUN + book getword detPendingCorrectedImfile $pageName det_id -var DET_ID + book getword detPendingCorrectedImfile $pageName exp_id -var EXP_ID + book getword detPendingCorrectedImfile $pageName class_id -var CLASS_ID + book getword detPendingCorrectedImfile $pageName exp_tag -var EXP_TAG XXX where does this come from + book getword detPendingCorrectedImfile $pageName det_type -var DET_TYPE XXX where does this come from + book getword detPendingCorrectedImfile $pageName camera -var CAMERA XXX where does this come from + book getword detPendingCorrectedImfile $pageName uri -var URI + book getword detPendingCorrectedImfile $pageName corr_id -var CORR_ID + book getword detPendingCorrectedImfile $pageName corr_type -var CORR_TYPE + book getword detPendingCorrectedImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingCorrectedImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detcorr.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID + + stdout $LOGDIR/detcorr.run.log + stderr $LOGDIR/detcorr.run.log + + $run = detrend_correct_imfile.pl --det_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --exp_tag $EXP_TAG --input_uri $URI --corr_id $CORR_ID --corr_type $CORR_TYPE --camera $CAMERA --outroot $outroot --redirect-output --verbose + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingCorrectedImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingCorrectedImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingCorrectedExp +# compare the new list with the ones already selected +task detrend.correctexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detcorr.exp.load.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tocorrectedexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingCorrectedExp_DB + command dettool -tocorrectedexp -limit 20 -dbname $DB:$detPendingCorrectedExp_DB + $detPendingCorrectedExp_DB ++ + if ($detPendingCorrectedExp_DB >= $DB:n) set detPendingCorrectedExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingCorrectedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingCorrectedExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingCorrectedExp + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.correctexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingCorrectedExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new exposures in detPendingCorrectedExp + book getpage detPendingCorrectedExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingCorrectedExp $pageName pantaskState RUN + book getword detPendingCorrectedExp $pageName det_id -var DET_ID + book getword detPendingCorrectedExp $pageName exp_id -var EXP_ID + book getword detPendingCorrectedExp $pageName det_type -var DET_TYPE + book getword detPendingCorrectedExp $pageName camera -var CAMERA + book getword detPendingCorrectedExp $pageName exp_tag -var EXP_TAG + book getword detPendingCorrectedExp $pageName corr_id -var CORR_ID + book getword detPendingCorrectedExp $pageName corr_type -var CORR_TYPE + book getword detPendingCorrectedExp $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingCorrectedExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $camera FPA + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detcorr.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID + + stdout $LOGDIR/detcorr.exp.run.log + stderr $LOGDIR/detcorr.exp.run.log + + $run = detrend_correct_exp.pl --det_id $DET_ID --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --corr_id $CORR_ID --corr_type $CORR_TYPE --camera $CAMERA --redirect-output --verbose + + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit detPendingCorrectedExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingCorrectedExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.mkruns.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.mkruns.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.mkruns.pro (revision 22160) @@ -0,0 +1,99 @@ +## detrend.mkruns.pro : globals and support macros : -*- sh -*- + +## XXX this script needs to be updated to use books, not queues +## XXX this script needs to be reconsidered... + +## This script defines tasks which regularly initiate the detrend +## runs. Each detrend type is given its own task, which are run once +## (or twice) per day to build the relevant detrend data + +## when one of these tasks is run, it creates a new detrend run ID in +## the database, and selects the exposures for that run on the basis +## of the specified filters + +# create a new detrend bias-creation run +task mkdetruns.megacam + # command dettool -select -time NOW -24h -type bias -camera megacam + command mkdetruns.megacam + host local + + periods -poll 1 + periods -exec 60 + periods -timeout 30 + trange 18:00 19:00 -nmax 1 + + # success: a new detrend run was created + task.exit 0 + echo "new MEGACAM detrend runs" + queueprint stdout + end + + # default exit status + task.exit default + echo "unable to define a new detrend run" + end + + # operation timed out? is the database down? + task.exit timeout + # send someone email? + echo "unable to define a new detrend run" + end +end + +# create a new detrend bias-creation run +task mkdetruns.isp + # command dettool -select -time NOW -24h -type bias -camera megacam + command mkdetruns.isp + host local + + periods -poll 1 + periods -exec 60 + periods -timeout 30 + trange 18:00 19:00 -nmax 1 + + # success: a new detrend run was created + task.exit 0 + echo "new ISP detrend runs" + queueprint stdout + end + + # default exit status + task.exit default + echo "unable to define a new detrend run" + end + + # operation timed out? is the database down? + task.exit timeout + # send someone email? + echo "unable to define a new detrend run" + end +end + +# create a new detrend bias-creation run +task mkdetruns.gpc + # command dettool -select -time NOW -24h -type bias -camera megacam + command mkdetruns.gpc + host local + + periods -poll 1 + periods -exec 60 + periods -timeout 30 + trange 18:00 19:00 -nmax 1 + + # success: a new detrend run was created + task.exit 0 + echo "new GPC detrend runs" + queueprint stdout + end + + # default exit status + task.exit default + echo "unable to define a new detrend run" + end + + # operation timed out? is the database down? + task.exit timeout + # send someone email? + echo "unable to define a new detrend run" + end +end Index: /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.norm.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.norm.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.norm.pro (revision 22160) @@ -0,0 +1,769 @@ +## detrend.norm.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend normalization stages +## these tasks use the books detPendingNormStatImfile detPendingNormImfile detPendingNormExp + +# test for required global variables +check.globals + +book init detPendingNormStatImfile +book init detPendingNormImfile +book init detPendingNormExp + +book init detCleanupNormStatImfile +book init detCleanupNormImfile +book init detCleanupNormExp + +macro detnorm.reset + book init detPendingNormStatImfile + book init detPendingNormImfile + book init detPendingNormExp + + book init detCleanupNormStatImfile + book init detCleanupNormImfile + book init detCleanupNormExp +end + +macro detnorm.status + book listbook detPendingNormStatImfile + book listbook detPendingNormImfile + book listbook detPendingNormExp + + book listbook detCleanupNormStatImfile + book listbook detCleanupNormImfile + book listbook detCleanupNormExp +end + +macro detnorm.on + task detrend.norm.load + active true + end + task detrend.norm.run + active true + end + task detrend.normexp.load + active true + end + task detrend.normexp.run + active true + end + task detrend.normstat.load + active true + end + task detrend.normstat.run + active true + end + + task detrend.cleanup.norm.load + active true + end + task detrend.cleanup.norm.run + active true + end + task detrend.cleanup.normexp.load + active true + end + task detrend.cleanup.normexp.run + active true + end + task detrend.cleanup.normstat.load + active true + end + task detrend.cleanup.normstat.run + active true + end +end + +macro detnorm.off + task detrend.norm.load + active false + end + task detrend.norm.run + active false + end + task detrend.normexp.load + active false + end + task detrend.normexp.run + active false + end + task detrend.normstat.load + active false + end + task detrend.normstat.run + active false + end + task detrend.cleanup.norm.load + active false + end + task detrend.cleanup.norm.run + active false + end + task detrend.cleanup.normexp.load + active false + end + task detrend.cleanup.normexp.run + active false + end + task detrend.cleanup.normstat.load + active false + end + task detrend.cleanup.normstat.run + active false + end +end + +# these variables will cycle through the known database names +$detPendingNormStatImfile_DB = 0 +$detPendingNormImfile_DB = 0 +$detPendingNormExp_DB = 0 + +$detCleanupNormStatImfile_DB = 0 +$detCleanupNormImfile_DB = 0 +$detCleanupNormExp_DB = 0 + +# select images ready for copy +# new entries are added to detPendingNormStatImfile +# compare the new list with the ones already selected +task detrend.normstat.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.normstat.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tonormalizedstat -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingNormStatImfile_DB + command dettool -tonormalizedstat -limit 20 -dbname $DB:$detPendingNormStatImfile_DB + $detPendingNormStatImfile_DB ++ + if ($detPendingNormStatImfile_DB >= $DB:n) set detPendingNormStatImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingNormStatImfile -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingNormStatImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingNormStatImfile + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.normstat.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingNormStatImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingNormStatImfile + book getpage detPendingNormStatImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingNormStatImfile $pageName pantaskState RUN + book getword detPendingNormStatImfile $pageName det_id -var DET_ID + book getword detPendingNormStatImfile $pageName det_type -var DET_TYPE + book getword detPendingNormStatImfile $pageName iteration -var ITERATION + book getword detPendingNormStatImfile $pageName camera -var CAMERA + book getword detPendingNormStatImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingNormStatImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s.%s.normstat.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION + + stdout $LOGDIR/detrend.normstat.log + stderr $LOGDIR/detrend.normstat.log + + $run = detrend_norm_calc.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE --outroot $outroot --redirect-output + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingNormStatImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingNormStatImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingNormImfile +# compare the new list with the ones already selected +task detrend.norm.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.norm.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tonormalize -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingNormImfile_DB + command dettool -tonormalize -limit 20 -dbname $DB:$detPendingNormImfile_DB + $detPendingNormImfile_DB ++ + if ($detPendingNormImfile_DB >= $DB:n) set detPendingNormImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingNormImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingNormImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingNormImfile + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.norm.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingNormImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingNormImfile + book getpage detPendingNormImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingNormImfile $pageName pantaskState RUN + book getword detPendingNormImfile $pageName det_type -var DET_TYPE + book getword detPendingNormImfile $pageName camera -var CAMERA + book getword detPendingNormImfile $pageName uri -var URI + book getword detPendingNormImfile $pageName det_id -var DET_ID + book getword detPendingNormImfile $pageName iteration -var ITERATION + book getword detPendingNormImfile $pageName class_id -var CLASS_ID + book getword detPendingNormImfile $pageName norm -var NORM + book getword detPendingNormImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingNormImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this stack + sprintf outroot "%s/%s.%s.%s/%s.%s.norm.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION + + stdout $LOGDIR/detrend.norm.log + stderr $LOGDIR/detrend.norm.log + + $run = detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE --outroot $outroot --redirect-output --verbose + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingNormImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingNormImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingNormExp +# compare the new list with the ones already selected +task detrend.normexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.normexp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tonormalizedexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingNormExp_DB + command dettool -tonormalizedexp -limit 20 -dbname $DB:$detPendingNormExp_DB + $detPendingNormExp_DB ++ + if ($detPendingNormExp_DB >= $DB:n) set detPendingNormExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingNormExp -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingNormExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingNormExp + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.normexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingNormExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingNormExp + book getpage detPendingNormExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingNormExp $pageName pantaskState RUN + book getword detPendingNormExp $pageName det_id -var DET_ID + book getword detPendingNormExp $pageName iteration -var ITERATION + book getword detPendingNormExp $pageName det_type -var DET_TYPE + book getword detPendingNormExp $pageName camera -var CAMERA + book getword detPendingNormExp $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingNormExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s.%s.normexp.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION + + stdout $LOGDIR/detrend.normexp.log + stderr $LOGDIR/detrend.normexp.log + + $run = detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE --outroot $outroot + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingNormExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingNormExp $options:0 pantaskState TIMEOUT + end +end + +########## cleanup normstat ########### +task detrend.cleanup.normstat.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.normstat.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -pendingcleanup_normalizedstat -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detCleanupNormStatImfile_DB + command dettool -pendingcleanup_normalizedstat -limit 20 -dbname $DB:$detCleanupNormStatImfile_DB + $detCleanupNormStatImfile_DB ++ + if ($detCleanupNormStatImfile_DB >= $DB:n) set detCleanupNormStatImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupNormStatImfile -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupNormStatImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupNormStatImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.normstat.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupNormStatImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupNormStatImfile (pantaskState == INIT) + book getpage detCleanupNormStatImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupNormStatImfile $pageName pantaskState RUN + book getword detCleanupNormStatImfile $pageName det_id -var DET_ID + book getword detCleanupNormStatImfile $pageName iteration -var ITERATION + book getword detCleanupNormStatImfile $pageName camera -var CAMERA + book getword detCleanupNormStatImfile $pageName state -var CLEANUP_MODE + book getword detCleanupNormStatImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.normstat.log + stderr $LOGDIR/detrend.cleanup.normstat.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.normstat.imfile --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupNormStatImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupNormStatImfile $options:0 pantaskState TIMEOUT + end +end + +########## cleanup norm (normalized.imfile) ########### +task detrend.cleanup.norm.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.norm.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -pendingcleanup_normalizedimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detCleanupNormImfile_DB + command dettool -pendingcleanup_normalizedimfile -limit 20 -dbname $DB:$detCleanupNormImfile_DB + $detCleanupNormImfile_DB ++ + if ($detCleanupNormImfile_DB >= $DB:n) set detCleanupNormImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupNormImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupNormImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupNormImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.norm.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupNormImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupNormImfile (pantaskState == INIT) + book getpage detCleanupNormImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupNormImfile $pageName pantaskState RUN + book getword detCleanupNormImfile $pageName det_id -var DET_ID + book getword detCleanupNormImfile $pageName iteration -var ITERATION + book getword detCleanupNormImfile $pageName class_id -var CLASS_ID + book getword detCleanupNormImfile $pageName camera -var CAMERA + book getword detCleanupNormImfile $pageName state -var CLEANUP_MODE + book getword detCleanupNormImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.norm.log + stderr $LOGDIR/detrend.cleanup.norm.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.norm.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupNormImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupNormImfile $options:0 pantaskState TIMEOUT + end +end + +########## cleanup normexp ########### +task detrend.cleanup.normexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.normexp.cleanup.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -pendingcleanup_normalizedexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detCleanupNormExp_DB + command dettool -pendingcleanup_normalizedexp -limit 20 -dbname $DB:$detCleanupNormExp_DB + $detCleanupNormExp_DB ++ + if ($detCleanupNormExp_DB >= $DB:n) set detCleanupNormExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupNormExp -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupNormExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupNormExp + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.normexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupNormExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupNormExp (pantaskState == INIT) + book getpage detCleanupNormExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupNormExp $pageName pantaskState RUN + book getword detCleanupNormExp $pageName det_id -var DET_ID + book getword detCleanupNormExp $pageName iteration -var ITERATION + book getword detCleanupNormExp $pageName camera -var CAMERA + book getword detCleanupNormExp $pageName state -var CLEANUP_MODE + book getword detCleanupNormExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.normexp.log + stderr $LOGDIR/detrend.cleanup.normexp.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.norm.exp --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupNormExp $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupNormExp $options:0 pantaskState TIMEOUT + end +end + Index: /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.process.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.process.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.process.pro (revision 22160) @@ -0,0 +1,532 @@ +## detrend.process.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend processing stage +## these tasks use the books detPendingProcessedImfile and detPendingProcessedExp + +# test for required global variables +check.globals + +book init detPendingProcessedImfile +book init detPendingProcessedExp +book init detCleanupProcessedImfile +book init detCleanupProcessedExp + +macro detproc.reset + book init detPendingProcessedImfile + book init detPendingProcessedExp + book init detCleanupProcessedImfile + book init detCleanupProcessedExp +end + +macro detproc.status + echo detPendingProcessedImfile + book listbook detPendingProcessedImfile + echo detPendingProcessedExp + book listbook detPendingProcessedExp + echo detCleanupProcessedImfile + book listbook detCleanupProcessedImfile + echo detCleanupProcessedExp + book listbook detCleanupProcessedExp +end + +macro detproc.on + task detrend.process.load + active true + end + task detrend.process.run + active true + end + task detrend.processexp.load + active true + end + task detrend.processexp.run + active true + end + task detrend.cleanup.process.load + active true + end + task detrend.cleanup.process.run + active true + end + task detrend.cleanup.processexp.load + active true + end + task detrend.cleanup.processexp.run + active true + end +end + +macro detproc.off + task detrend.process.load + active false + end + task detrend.process.run + active false + end + task detrend.processexp.load + active false + end + task detrend.processexp.run + active false + end + task detrend.cleanup.process.load + active false + end + task detrend.cleanup.process.run + active false + end + task detrend.cleanup.processexp.load + active false + end + task detrend.cleanup.processexp.run + active false + end +end + + +# these variables will cycle through the known database names +$detPendingProcessedImfile_DB = 0 +$detPendingProcessedExp_DB = 0 +$detCleanupProcessedImfile_DB = 0 +$detCleanupProcessedExp_DB = 0 + +# select images ready for copy +# new entries are added to detPendingProcessedImfile +# compare the new list with the ones already selected +task detrend.process.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.process.imfile.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -toprocessedimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingProcessedImfile_DB + command dettool -toprocessedimfile -limit 20 -dbname $DB:$detPendingProcessedImfile_DB + $detPendingProcessedImfile_DB ++ + if ($detPendingProcessedImfile_DB >= $DB:n) set detPendingProcessedImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingProcessedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingProcessedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingProcessedImfile + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand + end +end + +# run detrend_process_imfile.ps on pending images +task detrend.process.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingProcessedImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingProcessedImfile + book getpage detPendingProcessedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingProcessedImfile $pageName pantaskState RUN + book getword detPendingProcessedImfile $pageName det_id -var DET_ID + book getword detPendingProcessedImfile $pageName det_type -var DET_TYPE + book getword detPendingProcessedImfile $pageName exp_id -var EXP_ID + book getword detPendingProcessedImfile $pageName class_id -var CLASS_ID + book getword detPendingProcessedImfile $pageName exp_tag -var EXP_TAG + book getword detPendingProcessedImfile $pageName uri -var URI + book getword detPendingProcessedImfile $pageName camera -var CAMERA + book getword detPendingProcessedImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingProcessedImfile $pageName dbname -var DBNAME + book getword detPendingProcessedImfile $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # see chip.pro for examples + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detproc.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID + + stdout $LOGDIR/detrend.process.imfile.log + stderr $LOGDIR/detrend.process.imfile.log + + $run = detrend_process_imfile.pl --det_id $DET_ID --exp_id $EXP_ID --det_type $DET_TYPE --class_id $CLASS_ID --exp_tag $EXP_TAG --input_uri $URI --camera $CAMERA --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingProcessedImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingProcessedImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingProcessedExp +# compare the new list with the ones already selected +task detrend.processexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.process.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -toprocessedexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingProcessedExp_DB + command dettool -toprocessedexp -limit 20 -dbname $DB:$detPendingProcessedExp_DB + $detPendingProcessedExp_DB ++ + if ($detPendingProcessedExp_DB >= $DB:n) set detPendingProcessedExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingProcessedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingProcessedExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingProcessedExp + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.processexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingProcessedExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new exposures in detPendingProcessedExp + book getpage detPendingProcessedExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingProcessedExp $pageName pantaskState RUN + book getword detPendingProcessedExp $pageName det_id -var DET_ID + book getword detPendingProcessedExp $pageName exp_id -var EXP_ID + book getword detPendingProcessedExp $pageName det_type -var DET_TYPE + book getword detPendingProcessedExp $pageName camera -var CAMERA + book getword detPendingProcessedExp $pageName exp_tag -var EXP_TAG + book getword detPendingProcessedExp $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingProcessedExp $pageName dbname -var DBNAME + book getword detPendingProcessedExp $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detproc.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID + + stdout $LOGDIR/detrend.process.exp.log + stderr $LOGDIR/detrend.process.exp.log + + $run = detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output --verbose + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit detPendingProcessedExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingProcessedExp $options:0 pantaskState TIMEOUT + end +end + +######## cleanup process imfile ######## +task detrend.cleanup.process.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.process.imfile.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -pendingcleanup_processedimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detCleanupProcessedImfile_DB + command dettool -pendingcleanup_processedimfile -limit 20 -dbname $DB:$detCleanupProcessedImfile_DB + $detCleanupProcessedImfile_DB ++ + if ($detCleanupProcessedImfile_DB >= $DB:n) set detCleanupProcessedImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupProcessedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupProcessedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupProcessedImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.process.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupProcessedImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupProcessedImfile (pantaskState == INIT) + book getpage detCleanupProcessedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupProcessedImfile $pageName pantaskState RUN + book getword detCleanupProcessedImfile $pageName det_id -var DET_ID + book getword detCleanupProcessedImfile $pageName exp_id -var EXP_ID + book getword detCleanupProcessedImfile $pageName class_id -var CLASS_ID + book getword detCleanupProcessedImfile $pageName camera -var CAMERA + book getword detCleanupProcessedImfile $pageName state -var CLEANUP_MODE + book getword detCleanupProcessedImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.process.imfile.log + stderr $LOGDIR/detrend.cleanup.process.imfile.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.process.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupProcessedImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupProcessedImfile $options:0 pantaskState TIMEOUT + end +end + + +######## cleanup process exp ######## +task detrend.cleanup.processexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.process.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -pendingcleanup_processedexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detCleanupProcessedExp_DB + command dettool -pendingcleanup_processedexp -limit 20 -dbname $DB:$detCleanupProcessedExp_DB + $detCleanupProcessedExp_DB ++ + if ($detCleanupProcessedExp_DB >= $DB:n) set detCleanupProcessedExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupProcessedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupProcessedExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupProcessedExp + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.processexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupProcessedExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupProcessedExp (pantaskState == INIT) + book getpage detCleanupProcessedExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupProcessedExp $pageName pantaskState RUN + book getword detCleanupProcessedExp $pageName det_id -var DET_ID + book getword detCleanupProcessedExp $pageName exp_id -var EXP_ID + book getword detCleanupProcessedExp $pageName camera -var CAMERA + book getword detCleanupProcessedExp $pageName state -var CLEANUP_MODE + book getword detCleanupProcessedExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.process.exp.log + stderr $LOGDIR/detrend.cleanup.process.exp.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.process.exp --stage_id $DET_ID --exp_id $EXP_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupProcessedExp $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupProcessedExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.reject.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.reject.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.reject.pro (revision 22160) @@ -0,0 +1,148 @@ +## detrend.reject.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend processing stage +## these tasks use the book detRejectExp + +# test for required global variables +check.globals + +book init detRejectExp + +macro detreject.reset + book init detRejectExp +end + +macro detreject.status + book listbook detRejectExp +end + +macro detreject.on + task detrend.reject.load + active true + end + task detrend.reject.run + active true + end +end + +macro detreject.off + task detrend.reject.load + active false + end + task detrend.reject.run + active false + end +end + +# this variable will cycle through the known database names +$detRejectExp_DB = 0 + +# select images ready for copy +# new entries are added to detRejectExp +# compare the new list with the ones already selected +task detrend.reject.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.reject.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -todetrunsummary -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detRejectExp_DB + command dettool -todetrunsummary -limit 20 -dbname $DB:$detRejectExp_DB + $detRejectExp_DB ++ + if ($detRejectExp_DB >= $DB:n) set detRejectExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detRejectExp -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detRejectExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detRejectExp + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand failure + end +end + +# copy new images, sending job to desired host +task detrend.reject.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detRejectExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detRejectExp + book getpage detRejectExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detRejectExp $pageName pantaskState RUN + book getword detRejectExp $pageName det_id -var DET_ID + book getword detRejectExp $pageName iteration -var ITERATION + book getword detRejectExp $pageName det_type -var DET_TYPE + book getword detRejectExp $pageName mode -var MODE + book getword detRejectExp $pageName camera -var CAMERA + book getword detRejectExp $pageName workdir -var WORKDIR_TEMPLATE + book getword detRejectExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s.%s.%s.%s.detreject" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION + + stdout $LOGDIR/detrend.reject.log + stderr $LOGDIR/detrend.reject.log + + $run = detrend_reject_exp.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detRejectExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand failure + book setword detRejectExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.resid.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.resid.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.resid.pro (revision 22160) @@ -0,0 +1,533 @@ +## detrend.resid.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend processing stage +## these tasks use the books detPendingResidImfile and detPendingResidExp + +# test for required global variables +check.globals + +book init detPendingResidImfile +book init detPendingResidExp +book init detCleanupResidImfile +book init detCleanupResidExp + +macro detresid.reset + book init detPendingResidImfile + book init detPendingResidExp + book init detCleanupResidImfile + book init detCleanupResidExp +end + +macro detresid.status + book listbook detPendingResidImfile + book listbook detPendingResidExp + book listbook detCleanupResidImfile + book listbook detCleanupResidExp +end + +macro detresid.on + task detrend.resid.load + active true + end + task detrend.resid.run + active true + end + task detrend.residexp.load + active true + end + task detrend.residexp.run + active true + end + task detrend.cleanup.resid.load + active true + end + task detrend.cleanup.resid.run + active true + end + task detrend.cleanup.residexp.load + active true + end + task detrend.cleanup.residexp.run + active true + end +end + +macro detresid.off + task detrend.resid.load + active false + end + task detrend.resid.run + active false + end + task detrend.residexp.load + active false + end + task detrend.residexp.run + active false + end + task detrend.cleanup.resid.load + active false + end + task detrend.cleanup.resid.run + active false + end + task detrend.cleanup.residexp.load + active false + end + task detrend.cleanup.residexp.run + active false + end +end + +# these variables will cycle through the known database names +$detPendingResidImfile_DB = 0 +$detPendingResidExp_DB = 0 +$detCleanupResidImfile_DB = 0 +$detCleanupResidExp_DB = 0 + +# select images ready for copy +# new entries are added to detPendingResidImfile +# compare the new list with the ones already selected +task detrend.resid.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.resid.imfile.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -toresidimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingResidImfile_DB + command dettool -toresidimfile -limit 20 -dbname $DB:$detPendingResidImfile_DB + $detPendingResidImfile_DB ++ + if ($detPendingResidImfile_DB >= $DB:n) set detPendingResidImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingResidImfile -key det_id:iteration:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingResidImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingResidImfile + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.resid.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingResidImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingResidImfile + book getpage detPendingResidImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingResidImfile $pageName pantaskState RUN + book getword detPendingResidImfile $pageName det_id -var DET_ID + book getword detPendingResidImfile $pageName exp_id -var EXP_ID + book getword detPendingResidImfile $pageName iteration -var ITERATION + book getword detPendingResidImfile $pageName det_type -var DET_TYPE + book getword detPendingResidImfile $pageName mode -var MODE + book getword detPendingResidImfile $pageName exp_tag -var EXP_TAG + book getword detPendingResidImfile $pageName class_id -var CLASS_ID + book getword detPendingResidImfile $pageName uri -var URI + book getword detPendingResidImfile $pageName det_uri -var DET_URI + book getword detPendingResidImfile $pageName camera -var CAMERA + book getword detPendingResidImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingResidImfile $pageName dbname -var DBNAME + book getword detPendingResidImfile $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detresid.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID $ITERATION + + stdout $LOGDIR/detrend.resid.imfile.log + stderr $LOGDIR/detrend.resid.imfile.log + + $run = detrend_resid_imfile.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE --outroot $outroot --redirect-output --verbose + + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create command + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingResidImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingResidImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingResidExp +# compare the new list with the ones already selected +task detrend.residexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.resid.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -toresidexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingResidExp_DB + command dettool -toresidexp -limit 20 -dbname $DB:$detPendingResidExp_DB + $detPendingResidExp_DB ++ + if ($detPendingResidExp_DB >= $DB:n) set detPendingResidExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingResidExp -key det_id:iteration:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingResidExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingResidExp + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.residexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingResidExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingResidExp + book getpage detPendingResidExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingResidExp $pageName pantaskState RUN + book getword detPendingResidExp $pageName det_id -var DET_ID + book getword detPendingResidExp $pageName exp_id -var EXP_ID + book getword detPendingResidExp $pageName iteration -var ITERATION + book getword detPendingResidExp $pageName det_type -var DET_TYPE + book getword detPendingResidExp $pageName mode -var MODE + book getword detPendingResidExp $pageName exp_tag -var EXP_TAG + book getword detPendingResidExp $pageName include -var INCLUDE + book getword detPendingResidExp $pageName camera -var CAMERA + book getword detPendingResidExp $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingResidExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detresid.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID $ITERATION + + stdout $LOGDIR/detrend.resid.exp.log + stderr $LOGDIR/detrend.resid.exp.log + + $run = detrend_resid_exp.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output --verbose + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingResidExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingResidExp $options:0 pantaskState TIMEOUT + end +end + +######## cleanup resid imfile ######## +task detrend.cleanup.resid.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.resid.imfile.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -pendingcleanup_residimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detCleanupResidImfile_DB + command dettool -pendingcleanup_residimfile -limit 20 -dbname $DB:$detCleanupResidImfile_DB + $detCleanupResidImfile_DB ++ + if ($detCleanupResidImfile_DB >= $DB:n) set detCleanupResidImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupResidImfile -key det_id:iteration:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupResidImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupResidImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.resid.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupResidImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupResidImfile (pantaskState == INIT) + book getpage detCleanupResidImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupResidImfile $pageName pantaskState RUN + book getword detCleanupResidImfile $pageName det_id -var DET_ID + book getword detCleanupResidImfile $pageName exp_id -var EXP_ID + book getword detCleanupResidImfile $pageName class_id -var CLASS_ID + book getword detCleanupResidImfile $pageName iteration -var ITERATION + book getword detCleanupResidImfile $pageName camera -var CAMERA + book getword detCleanupResidImfile $pageName state -var CLEANUP_MODE + book getword detCleanupResidImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.resid.imfile.log + stderr $LOGDIR/detrend.cleanup.resid.imfile.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.resid.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupResidImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupResidImfile $options:0 pantaskState TIMEOUT + end +end + + +######## cleanup resid exp ######## +task detrend.cleanup.residexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.resid.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -pendingcleanup_residexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detCleanupResidExp_DB + command dettool -pendingcleanup_residexp -limit 20 -dbname $DB:$detCleanupResidExp_DB + $detCleanupResidExp_DB ++ + if ($detCleanupResidExp_DB >= $DB:n) set detCleanupResidExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupResidExp -key det_id:iteration:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupResidExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupResidExp + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.residexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupResidExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupResidExp (pantaskState == INIT) + book getpage detCleanupResidExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupResidExp $pageName pantaskState RUN + book getword detCleanupResidExp $pageName det_id -var DET_ID + book getword detCleanupResidExp $pageName exp_id -var EXP_ID + book getword detCleanupResidExp $pageName iteration -var ITERATION + book getword detCleanupResidExp $pageName camera -var CAMERA + book getword detCleanupResidExp $pageName state -var CLEANUP_MODE + book getword detCleanupResidExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.resid.exp.log + stderr $LOGDIR/detrend.cleanup.resid.exp.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.resid.exp --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupResidExp $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupResidExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.stack.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.stack.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/detrend.stack.pro (revision 22160) @@ -0,0 +1,276 @@ +## detrend.stack.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend stacking stage +## these tasks use the book detPendingStackedImfile + +# test for required global variables +check.globals + +book init detPendingStackedImfile +book init detCleanupStackedImfile + +macro detstack.reset + book init detPendingStackedImfile + book init detCleanupStackedImfile +end + +macro detstack.status + echo detPendingStackedImfile + book listbook detPendingStackedImfile + echo detCleanupStackedImfile + book listbook detCleanupStackedImfile +end + +macro detstack.on + task detrend.stack.load + active true + end + task detrend.stack.run + active true + end + task detrend.cleanup.stack.load + active true + end + task detrend.cleanup.stack.run + active true + end +end + +macro detstack.off + task detrend.stack.load + active false + end + task detrend.stack.run + active false + end + task detrend.cleanup.stack.load + active false + end + task detrend.cleanup.stack.run + active false + end +end + +# this variable will cycle through the known database names +$detPendingStackedImfile_DB = 0 +$detCleanupStackedImfile_DB = 0 + +# select images ready for detrend_stack.pl +# new entries are added to detPendingStackedImfile +# compare the new list with the ones already selected +task detrend.stack.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.stack.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tostacked -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingStackedImfile_DB + command dettool -tostacked -limit 20 -dbname $DB:$detPendingStackedImfile_DB + $detPendingStackedImfile_DB ++ + if ($detPendingStackedImfile_DB >= $DB:n) set detPendingStackedImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingStackedImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingStackedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingStackedImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.stack.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingStackedImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingStackedImfile + book getpage detPendingStackedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingStackedImfile $pageName pantaskState RUN + book getword detPendingStackedImfile $pageName det_id -var DET_ID + book getword detPendingStackedImfile $pageName iteration -var ITERATION + book getword detPendingStackedImfile $pageName det_type -var DET_TYPE + book getword detPendingStackedImfile $pageName class_id -var CLASS_ID + book getword detPendingStackedImfile $pageName camera -var CAMERA + book getword detPendingStackedImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingStackedImfile $pageName dbname -var DBNAME + book getword detPendingStackedImfile $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure + sprintf outroot "%s/%s.%s.%s/%s.%s.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION + + stdout $LOGDIR/detrend.stack.log + stderr $LOGDIR/detrend.stack.log + + $run = detrend_stack.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingStackedImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingStackedImfile $options:0 pantaskState TIMEOUT + end +end + +########## cleanup stack ########### +task detrend.cleanup.stack.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.stack.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -pendingcleanup_stacked -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detCleanupStackedImfile_DB + command dettool -pendingcleanup_stacked -limit 20 -dbname $DB:$detCleanupStackedImfile_DB + $detCleanupStackedImfile_DB ++ + if ($detCleanupStackedImfile_DB >= $DB:n) set detCleanupStackedImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupStackedImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupStackedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupStackedImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.stack.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupStackedImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupStackedImfile (pantaskState == INIT) + book getpage detCleanupStackedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupStackedImfile $pageName pantaskState RUN + book getword detCleanupStackedImfile $pageName det_id -var DET_ID + book getword detCleanupStackedImfile $pageName iteration -var ITERATION + book getword detCleanupStackedImfile $pageName class_id -var CLASS_ID + book getword detCleanupStackedImfile $pageName camera -var CAMERA + book getword detCleanupStackedImfile $pageName state -var CLEANUP_MODE + book getword detCleanupStackedImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.stack.log + stderr $LOGDIR/detrend.cleanup.stack.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.stack.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupStackedImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupStackedImfile $options:0 pantaskState TIMEOUT + end +end + Index: /branches/eam_branches/eam_branch_20080806/ippTasks/diff.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/diff.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/diff.pro (revision 22160) @@ -0,0 +1,286 @@ +## diff.pro : image difference analysis : -*- sh -*- + +## This file contains panTasks definitions for performing the image differencing. +## After a difference (with associated diff_id) is defined, the difference is performed +## (tasks in diffSkyfile). + +# test for required global variables +check.globals + +### Initialise the books containing the tasks to do +book init diffSkyfile +book init diffCleanup + +### Database lists +$diffSkycell_DB = 0 +$diffCleanup_DB = 0 + +### Check status of diffing tasks +macro diff.status + book listbook diffSkyfile + book listbook diffCleanup +end + +### Reset diffing tasks +macro diff.reset + book init diffSkyfile + book init diffCleanup +end + +### Turn diffing tasks on +macro diff.on + task diff.skycell.load + active true + end + task diff.skycell.run + active true + end + task diff.cleanup.load + active true + end + task diff.cleanup.run + active true + end +end + +### Turn diffing tasks off +macro diff.off + task diff.skycell.load + active false + end + task diff.skycell.run + active false + end + task diff.cleanup.load + active false + end + task diff.cleanup.run + active false + end +end + + +### Load tasks for doing the differences +### Tasks are loaded into diffSkyfile. +task diff.skycell.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/diff.skycell.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command difftool -todiffskyfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$diffSkycell_DB + command difftool -todiffskyfile -limit 20 -dbname $DB:$diffSkycell_DB + $diffSkycell_DB ++ + if ($diffSkycell_DB >= $DB:n) set diffSkycell_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout diffSkyfile -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook diffSkyfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup diffSkyfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the diff overlaps +### Tasks are taken from diffSkyfile. +task diff.skycell.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages diffSkyfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in diffSkyfile (pantaskState == INIT) + book getpage diffSkyfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword diffSkyfile $pageName pantaskState RUN + book getword diffSkyfile $pageName diff_id -var DIFF_ID + book getword diffSkyfile $pageName tess_id -var TESS_DIR + book getword diffSkyfile $pageName skycell_id -var SKYCELL_ID + book getword diffSkyfile $pageName camera -var CAMERA + book getword diffSkyfile $pageName workdir -var WORKDIR_TEMPLATE + book getword diffSkyfile $pageName dbname -var DBNAME + + ### XXX Instead of setting these by camera, need to set by skycell_id + ### XXX change these to use the --redirect scheme +# set.host.for.camera $CAMERA $DIFF_ID +# set.workdir.by.camera $CAMERA $DIFF_ID $WORKDIR_TEMPLATE $default_host WORKDIR + host anyhost + $WORKDIR = $WORKDIR_TEMPLATE + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.dif.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $DIFF_ID + + ## generate output log based on filerule (convert the URI to a PATH) + $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $DIFF_ID --basename $outroot` + if ("$logfile" == "") + echo "WARNING: logfile not defined in diff.skycell.run" + break + end + + stdout $logfile + stderr $logfile + dirname $logfile -var outpath + mkdir $outpath + + $run = diff_skycell.pl --diff_id $DIFF_ID --outroot $outroot + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit diffSkyfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword diffSkyfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for diff analysis +# new entries are added to diffPendingImfile +# skip already-present entries +task diff.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/diff.cleanup.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command difftool -pendingcleanuprun -limit 20 + else + # save the DB name for the exit tasks + option $DB:$diffCleanup_DB + command difftool -pendingcleanuprun -limit 20 -dbname $DB:$diffCleanup_DB + $diffCleanup_DB ++ + if ($diffCleanup_DB >= $DB:n) set diffCleanup_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout diffCleanup -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook diffCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup diffCleanup + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task diff.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages diffCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in diffCleanup (pantaskState == INIT) + book getpage diffCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword diffCleanup $pageName pantaskState RUN + book getword diffCleanup $pageName camera -var CAMERA + book getword diffCleanup $pageName state -var CLEANUP_MODE + book getword diffCleanup $pageName diff_id -var DIFF_ID + book getword diffCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and diff (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/diff.cleanup.log + stderr $LOGDIR/diff.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit diffCleanup $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword diffCleanup $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080806/ippTasks/fake.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/fake.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/fake.pro (revision 22160) @@ -0,0 +1,277 @@ +## fake.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the phase 0 stage +## these tasks use the book fakePendingImfile + +# test for required global variables +check.globals + +book init fakePendingImfile +book init fakePendingCleanup + +macro fake.status + book listbook fakePendingImfile + book listbook fakePendingCleanup +end + +macro fake.reset + book init fakePendingImfile + book init fakePendingCleanup +end + +macro fake.on + task fake.imfile.load + active true + end + task fake.imfile.run + active true + end + task fake.cleanup.load + active false + end + task fake.cleanup.run + active false + end +end + +macro fake.off + task fake.imfile.load + active false + end + task fake.imfile.run + active false + end + task fake.cleanup.load + active false + end + task fake.cleanup.run + active false + end +end + +# this variable will cycle through the known database names +$fakeImfile_DB = 0 + +# select images ready for fake analysis +# new entries are added to fakePendingImfile +# skip already-present entries +task fake.imfile.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/fake.imfile.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command faketool -pendingimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$fakeImfile_DB + command faketool -pendingimfile -limit 20 -dbname $DB:$fakeImfile_DB + $fakeImfile_DB ++ + if ($fakeImfile_DB >= $DB:n) set fakeImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout fakePendingImfile -key fake_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook fakePendingImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup fakePendingImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the fake_imfile.pl script on pending images +task fake.imfile.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages fakePendingImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in fakePendingImfile (pantaskState == INIT) + book getpage fakePendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fakePendingImfile $pageName pantaskState RUN + book getword fakePendingImfile $pageName camera -var CAMERA + book getword fakePendingImfile $pageName exp_id -var EXP_ID + book getword fakePendingImfile $pageName exp_tag -var EXP_TAG + book getword fakePendingImfile $pageName fake_id -var FAKE_ID + book getword fakePendingImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword fakePendingImfile $pageName class_id -var CLASS_ID + book getword fakePendingImfile $pageName chip_path_base -var CHIPROOT + book getword fakePendingImfile $pageName cam_path_base -var CAMROOT + book getword fakePendingImfile $pageName dbname -var DBNAME + book getword fakePendingImfile $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and fake (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s/%s.fk.%s" $WORKDIR $EXP_TAG $EXP_TAG $FAKE_ID + + stderr $LOGDIR/fake.imfile.log + stderr $LOGDIR/fake.imfile.log + + $run = fake_imfile.pl --exp_id $EXP_ID --fake_id $FAKE_ID --class_id $CLASS_ID --chiproot=$CHIPROOT --camroot=$CAMROOT --camera $CAMERA --outroot $outroot + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit fakePendingImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword fakePendingImfile $options:0 pantaskState TIMEOUT + end +end + +# this variable will cycle through the known database names +$fake_cleanup_DB = 0 + +# select images ready for fake analysis +# new entries are added to fakePendingImfile +# skip already-present entries +task fake.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/fake.cleanup.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command faketool -pendingcleanuprun -limit 20 + else + # save the DB name for the exit tasks + option $DB:$fake_cleanup_DB + command faketool -pendingcleanuprun -limit 20 -dbname $DB:$fake_cleanup_DB + $fake_cleanup_DB ++ + if ($fake_cleanup_DB >= $DB:n) set fake_cleanup_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout fakePendingCleanup -key fake_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook fakePendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup fakePendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task fake.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages fakePendingCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in fakePendingCleanup (pantaskState == INIT) + book getpage fakePendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fakePendingCleanup $pageName pantaskState RUN + book getword fakePendingCleanup $pageName camera -var CAMERA + book getword fakePendingCleanup $pageName state -var CLEANUP_MODE + book getword fakePendingCleanup $pageName fake_id -var FAKE_ID + book getword fakePendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and fake (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/fake.cleanup.log + stderr $LOGDIR/fake.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage fake --stage_id $FAKE_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit fakePendingCleanup $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword fakePendingCleanup $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080806/ippTasks/flatcorr.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/flatcorr.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/flatcorr.pro (revision 22160) @@ -0,0 +1,160 @@ +## flatcorr.pro : tasks to run flat-field correction : -*- sh -*- +## this file contains the tasks for running the flatcorr stage +## these tasks use the book 'flatcorrBook' + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/flatcorr +exec mkdir -p $LOGSUBDIR + +book init flatcoorBook + +macro flatcorr.reset + book init flatcorrBook +end + +macro flatcorr.status + book listbook flatcorrBook +end + +macro flatcorr.on + task flatcorr.init + active true + end +end + +macro flatcorr.off + task flatcorr.init + active false + end +end + +# these variables will cycle through the known ippdb database names +$calInit_DB = 0 + +# we have three steps here: +# 1) get the list of dvo databases: caltool -dbs +# 2) define a new flatcorr run for each of the dvo dbs +# caltool + +# create new flatcorr entries for the currently known DVO databases +# run this multiple times once an hour - on pass for each db +task flatcorr.load + host local + + # check the list of pending flatcorr runs + periods -poll 10 + periods -exec 900 + periods -timeout 60 + npending 1 + + # define the command (does not depend on previous queries) + if ($DB:n == 0) + command flatcorr -pending + else + # save the DB name for the exit tasks + # note that this DB name refers to the ippdb, not the dvodb + option $DB:$calInit_DB + command flatcorr -pending -dbname $DB:$calInit_DB + $calInit_DB ++ + if ($calInit_DB >= $DB:n) set calInit_DB = 0 + end + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/flatcorr.log + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + # XXX have ippTools report the dbname? + ipptool2book stdout flatcorrBook -key dbname:corr_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook flatcorrBook + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup flatcorrBook + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# create new flatcorr entries for the currently known DVO databases +# run this multiple times once an hour - one pass for each db +# can we include information that the db has been updated without a recent cal analysis? +task flatcorr.run + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 60 + # trange Hourly@00:00 Hourly@00:10 + # every hour on the hour + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/flatcorr.log + + task.exec + book npages flatcorrBook -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in flatcorrBook + # the sequencing in this task set is by the pantasksState (see notes.txt) + book getpage flatcorrBook 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword flatcorrBook $pageName pantaskState RUN + book setword flatcorrBook $pageName region $REGION + + # XXX probably need to set the output / log based on WORKDIR... + book getword flatcorrBook $pageName cal_id -var ID + book getword flatcorrBook $pageName dvodb -var DVODB + book getword flatcorrBook $pageName region -var REGION + book getword flatcorrBook $pageName filter -var FILTER + book getword flatcorrBook $pageName dbname -var DBNAME + + # specify choice of remote host + # set a specific DVO host here + if ($PARALLEL) + host anyhost + else + host local + end + + # save the pageName for future reference below + options $pageName + + $run = flatcorr_proc.pl --corr_id $ID --dvodb $DVODB --region $REGION --filter $FILTER + add_standard_args run + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit flatcorrBook $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword flatcorrBook $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080806/ippTasks/isp.det =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/isp.det (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/isp.det (revision 22160) @@ -0,0 +1,91 @@ + +detRunDef MULTI + +detRunDef METADATA + detRunName STR BIAS + prereq STR NONE + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/bias -det_type bias -select_exp_type bias -select_dateobs_begin 2007-01-13T00:00:00" + dbname STR isp + pantaskState STR DONE + detRunID STR 1 +END + +detRunDef METADATA + detRunName STR DARK-warm + prereq STR BIAS + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/dark -det_type dark -select_exp_type dark -select_exp_time_min 0.5 -select_ccd_temp_min -27.5 -select_ccd_temp_max -24.0 -ccd_temp_min -27.5 -ccd_temp_max -24.0" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR DARK-cool + prereq STR BIAS + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/dark -det_type dark -select_exp_type dark -select_exp_time_min 0.5 -select_ccd_temp_max -27.5 -ccd_temp_max -27.5" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR SHUTTER + prereq STR "DARK-warm DARK-cool" + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/shutter -det_type shutter -select_exp_type skyflat -select_exp_time_min 0.05 -select_filter g -select_dateobs_begin 2007-01-14T04:00:00 -select_dateobs_end 2007-01-14T05:00:00 -select_sat_pixel_frac_max 0.1" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-g + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/g -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter g -filter g -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-r + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/r -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter r -filter r -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-i + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/i -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter i -filter i -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-z + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/z -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter z -filter z -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-y + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/y -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter y -filter y -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FRINGE-z + prereq STR FLAT-z + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/fringe/z -det_type fringe -select_exp_type object -select_exp_time_min 0.05 -select_filter z -filter z -select_dateobs_begin 2007-01-14T09:19:00 -select_dateobs_end 2007-01-18" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FRINGE-y + prereq STR FLAT-y + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/fringe/y -det_type fringe -select_exp_type object -select_exp_time_min 0.05 -select_filter y -filter y -select_dateobs_begin 2007-01-13 -select_dateobs_end 2007-01-18" + dbname STR isp + pantaskState STR NULL +END Index: /branches/eam_branches/eam_branch_20080806/ippTasks/magic.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/magic.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/magic.pro (revision 22160) @@ -0,0 +1,419 @@ +## magic.pro : support for the streak removal : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/magic +exec mkdir -p $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init magicToTree +book init magicToProcess +book init magicToMask + +### Database lists +$magicTree_DB = 0 +$magicProcess_DB = 0 +$magicMask_DB = 0 + +### Check status of tasks +macro magic.status + book listbook magicToTree + book listbook magicToProcess + book listbook magicToMask +end + +### Reset tasks +macro magic.reset + book init magicTree + book init magicProcess + book init magicMask +end + +### Turn tasks on +macro magic.on + task magic.tree.load + active true + end + task magic.tree.run + active true + end + task magic.process.load + active true + end + task magic.process.run + active true + end + task magic.mask.load + active true + end + task magic.mask.run + active true + end +end + +### Turn tasks off +macro magic.on + task magic.tree.load + active false + end + task magic.tree.run + active false + end + task magic.process.load + active false + end + task magic.process.run + active false + end + task magic.mask.load + active false + end + task magic.mask.run + active false + end +end + +task magic.tree.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/magic.tree.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command magictool -totree -limit 20 + else + # save the DB name for the exit tasks + option $DB:$magicTree_DB + command magictool -totree -limit 20 -dbname $DB:$magicTree_DB + $magicTree_DB ++ + if ($magicTree_DB >= $DB:n) set magicTree_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicTree -key magic_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicTree + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicTree + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task magic.tree.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages magicTree -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicTree 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicTree $pageName pantaskState RUN + book getword magicTree $pageName magic_id -var MAGIC_ID + book getword magicTree $pageName exp_id -var EXP_ID + book getword magicTree $pageName camera -var CAMERA + book getword magicTree $pageName workdir -var WORKDIR_TEMPLATE + book getword magicTree $pageName dbname -var DBNAME + book getword magicTree $pageName tess_id -var TESS_DIR + book getword magicTree $pageName ra -var RA + book getword magicTree $pageName decl -var DEC + +# set.host.for.camera $CAMERA $MAGIC_ID +# set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR + host anyhost + $WORKDIR = $WORKDIR_TEMPLATE + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s.mgc.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID + + ## generate output log based on filerule (convert the URI to a PATH) + $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $MAGIC_ID --basename $outroot` + if ("$logfile" == "") + echo "WARNING: logfile not defined in magic.tree.run" + break + end + + stdout $logfile + stderr $logfile + dirname $logfile -var outpath + mkdir $outpath + + $run = magic_tree.pl --magic_id $MAGIC_ID --camera $CAMERA --tess_id $TESS_DIR --ra $RA --dec $DEC --outroot $outroot --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit magicTree $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicTree $options:0 pantaskState TIMEOUT + end +end + +task magic.process.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/magic.process.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command magictool -toprocess -limit 20 + else + # save the DB name for the exit tasks + option $DB:$magicTree_DB + command magictool -toprocess -limit 20 -dbname $DB:$magicProcess_DB + $magicProcess_DB ++ + if ($magicProcess_DB >= $DB:n) set magicProcess_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicProcess -key magic_id:node -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicProcess + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicProcess + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task magic.process.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages magicProcess -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicProcess 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicTree $pageName pantaskState RUN + book getword magicTree $pageName magic_id -var MAGIC_ID + book getword magicTree $pageName exp_id -var EXP_ID + book getword magicTree $pageName node -var NODE + book getword magicTree $pageName camera -var CAMERA + book getword magicTree $pageName workdir -var WORKDIR_TEMPLATE + book getword magicTree $pageName dbname -var DBNAME + +# set.host.for.camera $CAMERA $MAGIC_ID +# set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR + host anyhost + $WORKDIR = $WORKDIR_TEMPLATE + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s.mgc.%s.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID $NODE + + ## generate output log based on filerule (convert the URI to a PATH) + $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $MAGIC_ID --basename $outroot` + if ("$logfile" == "") + echo "WARNING: logfile not defined in magic.process.run" + break + end + + stdout $logfile + stderr $logfile + dirname $logfile -var outpath + mkdir $outpath + + $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --outroot $outroot --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit magicProcess $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicProcess $options:0 pantaskState TIMEOUT + end +end + +task magic.mask.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/magic.mask.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command magictool -tomask -limit 20 + else + # save the DB name for the exit tasks + option $DB:$magicMask_DB + command magictool -tomask -limit 20 -dbname $DB:$magicMask_DB + $magicMask_DB ++ + if ($magicMask_DB >= $DB:n) set magicMask_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicMask -key magic_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicMask + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicMask + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task magic.mask.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages magicMask -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicMask 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicMask $pageName pantaskState RUN + book getword magicMask $pageName magic_id -var MAGIC_ID + book getword magicMask $pageName exp_id -var EXP_ID + book getword magicMask $pageName camera -var CAMERA + book getword magicMask $pageName workdir -var WORKDIR_TEMPLATE + book getword magicMask $pageName dbname -var DBNAME + +# set.host.for.camera $CAMERA $MAGIC_ID +# set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR + host anyhost + $WORKDIR = $WORKDIR_TEMPLATE + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s.mgc.%s.mask" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID + + ## generate output log based on filerule (convert the URI to a PATH) + $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $MAGIC_ID --basename $outroot` + if ("$logfile" == "") + echo "WARNING: logfile not defined in magic.tree.run" + break + end + + stdout $logfile + stderr $logfile + dirname $logfile -var outpath + mkdir $outpath + + $run = magic_mask.pl --magic_id $MAGIC_ID --camera $CAMERA --outroot $outroot --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit magicMask $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicMask $options:0 pantaskState TIMEOUT + end +end + Index: /branches/eam_branches/eam_branch_20080806/ippTasks/mkdetrend.config =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/mkdetrend.config (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/mkdetrend.config (revision 22160) @@ -0,0 +1,43 @@ + +detRunDef MULTI + +# add in the min number of input images and a time range +detRunDef METADATA + detRunName STR BIAS + prereq STR NONE + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type BIAS -select_exp_type BIAS" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR DARK + prereq STR BIAS + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type DARK -select_exp_type DARK" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR SHUTTER + prereq STR DARK + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-r + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-i + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter i -select_exp_type FLAT -select_filter i" + dbname STR eamtest + pantaskState STR NULL +END Index: /branches/eam_branches/eam_branch_20080806/ippTasks/mkdetrend.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/mkdetrend.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/mkdetrend.pro (revision 22160) @@ -0,0 +1,217 @@ +## mkdetrend.pro : autorun the detrend analysis : -*- sh -*- + +# detRun METADATA +# det_id STR 3 +# iteration S32 0 +# det_type STR SHUTTER +# mode STR master +# state STR run +# filelevel STR FPA +# END + +macro load.detRunDefs + if ($0 != 2) + echo "USAGE: load.detRunDefs (filename)" + break + end + queueload detruns -x "cat $MODULES:0/$1" + ipptool2book detruns detRunDef -key detRunName -uniq -setword pantaskState INIT + queuedelete detruns +end + +task detrun.launch + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + # each outstanding detrend run blocks the next one. only launch the next + # if the current one is done + task.exec + local i j N found missed detRunName prereq prereqList prereqState cmdOptions dbname + + # how many detRunDefs are waiting to be started? + book npages detRunDef -var N -key pantaskState INIT + # echo "detrun.launch N: $N" + if ($N == 0) break + + $found = 0 + + # search the detRunDef book for an entry which is unstarted (state NULL) + # and for which the dependencies are done + for i 0 $N + book getpage detRunDef $i -var detRunName -key pantaskState INIT + # echo "detrun.launch detRunName: $detRunName" + if ("$detRunName" == "NULL") break + + book getword detRunDef $detRunName prereq -var prereqList + if (("$prereqList" == "NULL") || ("$prereqList" == "NONE")) + $found = 1 + last + end + + list prereq -split $prereqList + $missed = 0 + for j 0 $prereq:n + # prereq must exist and be in state of DONE + book getword detRunDef $prereq:$j pantaskState -var prereqState + + # echo "detrun.launch prereqState: $prereqState" + # echo "detrun.launch detRunName: $detRunName" + if ("$prereqState" != "DONE") + $missed = 1 + last + end + end + if ($missed == 0) + $found = 1 + last + end + + if ($found == 0) + if ($VERBOSE >= 2) + echo "$detRunName not ready" + end + else + if ($VERBOSE >= 2) + echo "$detRunName ready" + end + end + end + + if ($found == 0) break + + # echo "detrun.launch detRunName: $detRunName" + + book getword detRunDef $detRunName options -var cmdOptions + book getword detRunDef $detRunName dbname -var dbname + book setword detRunDef $detRunName pantaskState INIT + + if ($VERBOSE >= 1) + echo "starting detrend $detRunName :" + echo "command dettool $cmdOptions -dbname $dbname" + end + + options $dbname $detRunName + command dettool $cmdOptions -dbname $dbname + end + + task.exit $EXIT_SUCCESS + local detRunID + + $dbname = $options:0 + $detRunName = $options:1 + + # convert 'stdout' to book format + ipptool2book stdout detRun -key det_id -uniq + if ($VERBOSE > 2) + book listbook detRun + end + + # we should have launched only a single detRun + book getpage detRun 0 -var detRunID + if ("$detRunID" == "NULL") + echo "ERROR:detRun not defined" + break + end + + # we need to track the detID generated + book setword detRunDef $detRunName detRunID $detRunID + book setword detRunDef $detRunName pantaskState RUN + + # drop the detRun book after we've grabbed the state + book delete detRun + end +end + +$detRunNcheck = 0 + +# loop over the outstanding detRuns and update state +task detrun.check + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + # each outstanding detrend run blocks the next one. only launch the next + # if the current one is done + task.exec + local i N found detRunName detRunID dbname + + book npages detRunDef -var N -key pantaskState RUN + if ($N == 0) break + if ($detRunNcheck >= $N) + $detRunNcheck = 0 + end + + # try the next detRun entry which is in state RUN + book getpage detRunDef $detRunNcheck -var detRunName -key pantaskState RUN + $detRunNcheck ++ + + if ("$detRunName" == "NULL") + if ($VERBOSE >= 3) + echo "no more active detruns (Ncheck = $detRunNcheck)" + end + break + end + + # info needed to check on current state + book getword detRunDef $detRunName detRunID -var detRunID + book getword detRunDef $detRunName dbname -var dbname + + if ($VERBOSE >= 2) + echo " checking on $detRunName" + echo "command dettool -runs -det_id $detRunID -dbname $dbname" + end + + # we need a dettool command which takes a specific det_id and returns + # the current state + options $dbname $detRunName + command dettool -runs -det_id $detRunID -dbname $dbname + end + + task.exit $EXIT_SUCCESS + local detRunID state + + $dbname = $options:0 + $detRunName = $options:1 + + # convert 'stdout' to book format + ipptool2book stdout detRun -key det_id -uniq + if ($VERBOSE > 2) + book listbook detRun + end + + # we should have launched only a single detRun + book getpage detRun 0 -var detRunID + if ("$detRunID" == "NULL") + echo "ERROR:detRun not defined" + break + end + + # what is the current state of the detRun? + book getword detRun $detRunID state -var state + + # use any other state? + if ("$state" == "stop") + # update the state for this detRunName, drop detRun page + book setword detRunDef $detRunName pantaskState DONE + if ($VERBOSE > 1) + echo "$detRunName is done" + end + else + if ($VERBOSE > 1) + echo "$detRunName is still running" + end + end + + # drop this book when completed + book delete detRun + end +end Index: /branches/eam_branches/eam_branch_20080806/ippTasks/notes.txt =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/notes.txt (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/notes.txt (revision 22160) @@ -0,0 +1,314 @@ + + proposal to clean scripts/tasks up somewhat wrt names: + + * scripts do NOT use workdir to change the workdir in the db + * thus, what is supplied is actually the outputRoot + * require the output root for the scripts + * have the calling tasks construct the outputRoot + ** replace -workdir with -outroot + + some filename rules : + + * the db entry WORKDIR is the most abstract version: + path://GPC-@HOST@/20080130 + file://data/@HOST@.0/gpc1/20080130 + neb:///@HOST@/gpc1/20080130 + neb://gpc1/20080130 + XXX have nebulous distinguish between neb:///host/path and + neb://path? + + * the db WORKDIR entry is carried from step to step unless + over-ridden with a -queue command + + * psLib-based binaries (ppImage, etc) may be passed URIs, but not + files with the @HOST@ abstraction + + * IPP perl scripts may be passed URIs, but not files with the @HOST@ + abstraction + + * nebulous : convert neb:///@HOST@/path/name + + *** examples: + + * this tells ppImage to use the ipp004 version for both input and output: + ppImage -file neb:///ipp004-v0/gpc1/20080130/o4414g0001o/o4414g0001o.XY01.fits neb:///ipp004-v0/gpc1/20080130/o4414g0001o/o4414g0001o.XY01 + + * we already know the hostname at this point; include it in the call + chip_imfile.pl --uri neb:///ipp004-v0/gpc1/20080130/o4414g0001o/o4414g0001o.XY01.fits --workdir neb:///ipp004-v0/gpc1/20080130/ + + + + + +2008.01.21 + + I'm working on the summit.copy.pro interaction with nebulous (and + those of the other ippTasks) and the assignment of jobs and data + locations to hosts on the basis of the chip/host relationship. I + have defined a table which relates the chip (class_id) to the + desired host name. There are a few places, and a few slightly + different ways, in which this gets used: + + * choice of host for command execution: + + every job which operates on a specific chip must be told the chip + host so it can run the job on that machine. the information is + supplied in pantasks to the job with the 'host' command. I've + made a pantasks macro 'set.host.by.camera' which takes the camera + and class_id, looks up the match in the chip/host table, and calls + the host command with the desired host. If there is no match + (camera not listed in chip/host table or chip not listed in + table), the host is set to 'anyhost', which randomly chooses one. + I have used this command for all tasks which generate a pcontrol + command. For the tasks which are performed at the camera level, + these are set to anyhost. For the tasks which operate on stacks, + we can use the same structure to associate the skycell with a + host. + + * specific output filenames + + In both ippTasks and ippScripts, there are references to files + which must correctly located on the host. + + * workdir choice of output directory. + + the + + * dsget output target: + + dsget is called by pantasks in the task 'dsget'. currently, dsget + accepts a --filename argument which specifies the output filename + to use. in the nebulous context, this is of the form + neb://top/next/stuff, without any host-specific information. To + target the file instance to a specific host, we pass dsget the + --volume (name) argument. The volumes refer to the specific + devices (potentially more than one per host), and the names can + then be chosen to include the host. + +2007.10.11 + + The calibration suite uses a database table of known/registered DVO + databases to be calibrated. The suite operates on a segment of the + sky in each run, nominally a section ~1h west of the current ST. + + caltool -pendingdbs returns a list of entries (mdc format), one for + each DVO database in the table. + + caltool -databases [-active] : returns the list of dvo databases + caltool -addcalrun : inserts the stats for a single calibration run + caltool -setstate (active/inactive) : modify the state of a dvo + database + caltool -adddatabase : insert a new database + + XXX a potential source of collisions. the books are marked by + unique keys in the db tables, but a single pantasks run may be + addressing multiple databases. IDs are not unique across + databases. we can solve this by including in the key the dbname + + the calibration suite runs the following tasks, in sequences on each + of the databases: + + * resort, relphot, uniphot, relastro.objects, relastro.images + + ** detrend correction: any detrend images which need to be corrected + get inserted in the detCorrectedExp table. + + dettool commands needed: + * add a detrend exposure (det_id, exp_id) to the detCorrectedExp table + * add a detrend imfile (det_id, exp_id, class_id) to the + detCorrectedImfile table + * select pending detCorrectedExp + * select pending detCorrectedImfiles + + +2007.01.08 + + I have updated pantasks to used named tags in named groups to make + life much easier when changing the schema of the db tables. The + function ipptool2book convernts the output from an ippTool query in + the form of a set of metadata structures (saved in stdout) into a + book format. + + dettool -tonormalizedstat DetrendNormStat + dettool -tonormalize DetrendNorm + dettool -tonormalizedexp DetrendNormExp + dettool -toprocessedimfile DetrendProcessImfiles + dettool -toprocessedexp DetrendProcessExposures + dettool -residdetrun DetrendRejectExp + dettool -toresidimfile DetrendResidImfiles + dettool -toresidexp DetrendResidExposures + dettool -tostacked DetrendStackClass + p0tool -pendingimfile Phase0Imfiles + p0tool -pendingexp Phase0Exposures + p2tool -pendingimfile Phase2Imfiles + p3tool -pendingexp Phase3Exposures + +2006.08.28 + +- ipptools need to use expID:camera:telescope as unique key (ie, not expID alone) +- p0tool -pendingimfile -simple needs to report the expKey (ie, not expID) +- + + +This directory contains the PanTasks scripts which define the steps +performed by the IPP. These scripts use the ippTools to decide what +images need to be processed, and to update the Metadata Database with +the current state of the analysis. The analysis is performed using +IPP programs which are distributed to the cluster computers as needed. + +1. The Detrend Creation Scripts + +The detrend creation process is among the more complex elements of the +IPP. There are N major steps which must be performed: + +1.1. Define New Detrend Runs + +The script 'detrend.mkruns.pro' defines tasks which initiate new +Detrend Runs. There is a separate task for each camera automatically +processed by the system (mkdetruns.CAMERA). These tasks are executed +on a regular, but infrequent basis. The tasks call external helper +programs which actually run the command dettools -define. The helper +programs are thus responsible for identifying all of the types of +detrend images to be run for a given camera, as well as the correct +set of filters to apply. These tasks are run locally and only affect +the database. PanTasks does not track the detrend runs which are +created. + +- we may need to define these within the PanTasks scripts +- how frequently are these tasks run (1/day?) +- if no raw images are selected by the query, no new detrun should be + created. +- we may need different schedules for the dark (bias/dark) and the + light (flat/fringe) data. And yet another schedule for sky data + (skyflat/skyfringe). + +1.2. Process new raw imfiles + +The script 'detrend.process.pro' defines tasks which processes the new +raw imfiles for any detrend run. This script defines two tasks, +'dettool.raw.load' and 'dettool.raw.process'. The first script +occasionally polls the Metadata Database to select raw imfiles which +have been identified as part of a detrend run. The identified raw +imfiles are added to an internal queue, DetrendImfilesToProcess. Any +raw imfile already in the list is ignored. The imfiles are added to +the queue with a state of NEW. The second task examines the queue for +imfiles with state NEW and defines a ppImage analysis job for that +imfile. The job is sent to pcontrol for processing on the distributed +cluster. At this point, the corresponding queue entry is set to the +state RUN. Upon success, the queue entry is set to DONE and the +database is notified of the successful operation. The DONE entries +are occasionally pruned from the queue; this must be done in the same +task as the initial query and the update of the list based on the +database results. + +- we need to define the columns in the output of dettool -raw +- we need a method to associate an imfile with a node in the cluster +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs +- we need a method to identify the correct recipe (function of type?) + +1.3. Stack new processed imfiles + +The script 'detrend.stack.pro' defines tasks which stack the newly +processed imfiles for any detrend run iteration. This script defines +two tasks, 'dettool.stack.load and 'dettool.stack.process. The first +script occasionally polls the Metadata Database to select class IDs +for detrend runs for which the input imfiles have all been processed. +The identified detRun/classID combination is added to an internal +queue, DetrendClassIDtoStack. The entries are added to the queue with +a state of NEW. The second task examines the queue for entries with a +state of NEW. It defines a ppMerge analysis job for that +detID/classID. The helper script selects the corresponding input +imfiles and passes them to ppMerge. The job is sent to pcontrol for +processing on the distributed cluster. At this point, the +corresponding queue entry is set to the state RUN. Upon success, the +queue entry is set to DONE and the database is notified of the +successful operation. The DONE entries are occasionally pruned from +the queue; this must be done in the same task as the initial query and +the update of the list based on the database results. + +- we need to define the columns in the output of dettool -chip -processed +- we need a method to associate a classID with a node in the cluster +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) +- we need a method to identify the correct recipe (function of type?) + + +1.4. Normalize stacked masters + +The script 'detrend.norm.pro' defines tasks which normalize the newly +processed stacks for any detrend run iteration. This script defines +two tasks, 'dettool.norm.load and 'dettool.norm.process. The first +script occasionally polls the Metadata Database to select detrun IDs +for which the classes have all been stacked. The identified +detRunID/iteration combination is added to an internal queue, +DetrendRunToNormalize. The entries are added to the queue with a +state of NEW. The second task examines the queue for entries with a +state of NEW. It defines a ppNorm analysis job for that +detID/iteration. The job is sent to pcontrol for processing on the +distributed cluster. At this point, the corresponding queue entry is +set to the state RUN. Upon success, the queue entry is set to DONE and +the database is notified of the successful operation. The DONE +entries are occasionally pruned from the queue; this must be done in +the same task as the initial query and the update of the list based on +the database results. + +- we need to define the columns in the output of dettool +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) +- we need a method to identify the correct recipe (function of type?) +- we need to define the ppNorm operation inputs and outputs better + + +1.5. Generate Resid Imfiles + +The script 'detrend.resid.pro' defines tasks which apply the newly +generated detrend image to the corresponding input images to generate +the residual images. This script defines two tasks, +'dettool.resid.load and 'dettool.resid.process. The first script +occasionally polls the Metadata Database to select imfiles for which +the master detrend images has been normalized. The identified imfiles +are added to an internal queue, DetrendResidImfiles. The entries are +added to the queue with a state of NEW. The second task examines the +queue for entries with a state of NEW. It defines a ppImage analysis +job for that imfile. The job is sent to pcontrol for processing on +the distributed cluster. At this point, the corresponding queue entry +is set to the state RUN. Upon success, the queue entry is set to DONE +and the database is notified of the successful operation. The DONE +entries are occasionally pruned from the queue; this must be done in +the same task as the initial query and the update of the list based on +the database results. + +- we need to define the columns in the output of dettool +- can we select the imfiles here without apriori knowledge that the + master has been normalized? +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) +- we need a method to identify the correct recipe (function of type?) +- we need a method to pass the matching detrend image to ppImage in + the correct context (ie, a bias needs to be supplied to the -bias element...) + +1.6. Assess Detrend Results + +The script 'detrend.assess.pro' defines tasks which assess the +residual statistics for completed detrend runs. This script defines +two tasks, 'dettool.assess.load and 'dettool.assess.process. The +first script occasionally polls the Metadata Database to select +detrunIDs/iterations for which the resid imfiles have all been +generated. The identified detruns are added to an internal queue, +DetrendRunToAssess. The entries are added to the queue with a state +of NEW. The second task examines the queue for entries with a state +of NEW. It defines an analysis job for that collection. The job is +sent to pcontrol for processing on the distributed cluster. At this +point, the corresponding queue entry is set to the state RUN. Upon +success, the queue entry is set to DONE and the database is notified +of the successful operation. The DONE entries are occasionally pruned +from the queue; this must be done in the same task as the initial +query and the update of the list based on the database results. + +- the assess tool needs to trigger a new run, if needed. +- what is the command to run on the resid statistics? +- we need to define the columns in the output of dettool +- can we select the detrend runs here? +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) Index: /branches/eam_branches/eam_branch_20080806/ippTasks/pantasks.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/pantasks.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/pantasks.pro (revision 22160) @@ -0,0 +1,435 @@ +## pantasks.pro : globals and support macros : -*- sh -*- + +# globals used to control system behavior : these should be in uppercase +$NEBULOUS = 0 +$NETWORK = 1 +$PARALLEL = 1 +$VERBOSE = 1 +$LOGDIR = `pwd` +$LOGDIR = $LOGDIR/pantasks_logs +mkdir $LOGDIR +$KEEP_FAILURES = 0 + +$LOADPOLL = 0.25 +$LOADEXEC = 5 +$RUNPOLL = 0.5 +$RUNEXEC = 1 + +$EXIT_SUCCESS = 0 +$EXIT_UNKNOWN_ERR = 1 +$EXIT_SYS_ERR = 2 +$EXIT_CONFIG_ERR = 3 +$EXIT_PROG_ERR = 4 +$EXIT_DATA_ERR = 5 + +# DB lists the database names in use; the default ipprc database +# is only used if DB:n is 0 +$DB:n = 0 + +# very basic values: set these with init.copy.mhpcc +$default_host = anyhost +$workdir_template = `pwd` + +# user function to add databases by name (avoids duplicates) +macro add.database + if ($0 != 2) + echo "USAGE: add.database (db)" + break + end + if ($?DB:n == 0) + list DB -add $1 + return + end + + $found = 0 + for i 0 $DB:n + if ($DB:$i == $1) + $found = 1 + echo "$DB:$i set" + last + end + end + + if ($found == 0) + list DB -add $1 + end +end + +macro del.database + if ($0 != 2) + echo "USAGE: del.database (db)" + break + end + if ($?DB:n == 0) + return + end + + list DB -del $1 +end + +macro init.isp + list DB -add isp +end + +macro init.essence + list DB -add essence_v2 +end + +macro init.simtest.local + list DB -add simtest + $PARALLEL = 0 + controller exit true +end + +macro init.simtest.default.db + $host = `hostname` + if ($PARALLEL) + controller exit true + controller host add $host + end +end + +macro init.simtest + list DB -add simtest + + $host = `hostname` + if ($PARALLEL) + controller exit true + controller host add $host + end +end + +# need the ability to activate or deactivate specific tasks... + +macro detrend.on + detproc.on + detstack.on + detnorm.on + detresid.on + detreject.on +end + +macro detrend.off + detproc.off + detstack.off + detnorm.off + detresid.off + detreject.off +end + +macro detrend.reset + detproc.reset + detstack.reset + detnorm.reset + detresid.reset + detreject.reset +end + +macro all.on + register.on + detrend.on + chip.on + camera.on + fake.on + warp.on + diff.on + stack.on +end + +macro all.off + register.off + detrend.off + chip.off + camera.off + fake.off + warp.off + diff.off + stack.off +end + +macro module.tasks + module register.pro + module detrend.process.pro + module detrend.stack.pro + module detrend.norm.pro + module detrend.resid.pro + module detrend.reject.pro + module chip.pro + module camera.pro + module fake.pro + module warp.pro + module diff.pro + module stack.pro +end + +macro showcommand + local n + + if ($VERBOSE) + $command = $taskarg:0 + for n 1 $taskarg:n + $command = $command $taskarg:$n + end + if ($0 == 2) + echo "" + echo "$1 for: $command" + echo "job exit status: $JOB_STATUS" + echo "job host: $JOB_HOSTNAME" + echo "job dtime: $JOB_DTIME" + else + echo "command: $command" + end + end +end + +macro add_standard_args + if ($0 != 2) + echo "Must pass in the command of interest, and this function will supplement" + stop + end + + local command + $command = $$1 + + if ($?DBNAME && "$DBNAME" != "DEFAULT") + $command = $command --dbname $DBNAME + end + if ($?NOOP != 0) + if ($NOOP != 0) + $command = $command --no-op + end + end + if ($?NOUPDATE != 0) + if ($NOUPDATE != 0) + $command = $command --no-update + end + end + $$1 = $command --verbose +end + + +macro process_exit + if ($0 != 4) + echo "USAGE: process_exit (bookName) (pageName) (exitCode)" + break + end + + $bookName = $1 + $pageName = $2 + $exitCode = $3 + + if ($VERBOSE > 4) + echo "*** exit status ***" + echo "JOB_STATUS: $JOB_STATUS" + echo "JOB_HOSTNAME: $JOB_HOSTNAME" + echo "JOB_DTIME: $JOB_DTIME" + echo "*** stdout ***" + queueprint stdout + echo "*** stderr ***" + queueprint stderr + end + + # success + if ($exitCode == $EXIT_SUCCESS) + # the handler scripts update DB the tables; here we just update the page + book setword $bookName $pageName pantaskState DONE + return + end + + # failure related to the data files + # jobs which result in DATAERR must have db pantaskState updated + if ($exitCode == $EXIT_DATA_ERR) + showcommand failure + book setword $bookName $pageName pantaskState DATA_ERR + return + end + + if ($VERBOSE) + echo "*** stdout ***" + queueprint stdout + echo "*** stderr ***" + queueprint stderr + end + + # failure related to the data files + if ($exitCode == $EXIT_SYS_ERR) + # stop + showcommand "system failure" + book setword $bookName $pageName pantaskState SYS_ERR + return + end + + # failure related to the data files + if ($exitCode == $EXIT_CONFIG_ERR) + # stop + showcommand "config error" + book setword $bookName $pageName pantaskState CONFIG_ERR + return + end + + # failure related to the data files + if ($exitCode == $EXIT_PROG_ERR) + # stop + showcommand "programming error" + book setword $bookName $pageName pantaskState PROG_ERR + return + end + + # any other exit status + showcommand failure + book setword $bookName $pageName pantaskState UNKNOWN_ERR +end + +## XXX for the moment, remove all errors +macro process_cleanup + if ($0 != 2) + echo "USAGE: process_cleanup (bookname)" + break + end + + book delpage $1 -key pantaskState DONE + + if (not($KEEP_FAILURES)) + book delpage $1 -key pantaskState SYS_ERR + book delpage $1 -key pantaskState DATA_ERR + book delpage $1 -key pantaskState PROG_ERR + book delpage $1 -key pantaskState CONFIG_ERR + book delpage $1 -key pantaskState UNKNOWN_ERR + end +end + +macro set.verbosity + if ($0 != 2) + echo "USAGE: set.verbosity (level)" + break + end + $VERBOSE = $1 +end + +macro show.variables + ?? +end + +macro set.workdir.by.camera + if ($0 != 6) + echo "USAGE: set.workdir.by.camera (camera) (class_id) (template) (default) (varname)" + break + end + + local host default template camera classID varname + $camera = $1 + $classID = $2 + $template = $3 + $default = $4 + $varname = $5 + + if ("$template" == "NULL") + $varname = $workdir_template + echo "WARNING: WORKDIR template not set. Defaulting to $workdir_template" + break + end + + # missing camera and/or chiphosts table results in host = NULL + book getword chiphosts $camera $classID -var host + + if ("$host" == "NULL") + strsub $template @HOST@ $default -var $varname + else + strsub $template @HOST@ $host -var $varname + end +end + +macro set.volume.by.camera + if ($0 != 3) + echo "USAGE: set.volume.by.camera (camera) (class_id) (template) (default) (varname)" + echo " uses global variable NEBULOUS to determine if nebulous is active" + break + end + + local host default template camera classID varname + $camera = $1 + $classID = $2 + $template = $3 + $default = $4 + $varname = $5 + + # missing camera and/or chiphosts table results in host = NULL + book getword chiphosts $camera $classID -var host + + if (not($NEBULOUS)) + $$varname = "NULL" + return + end + + if ("$host" == "NULL") + $$varname = $default + return + end + + strsub $template @HOST@ $host -var $varname +end + +macro set.host.for.camera + if ($0 != 3) + echo "USAGE: set.host.for.camera (camera) (class_id)" + break + end + + if (not($PARALLEL)) + host local + return + end + + # missing camera and/or chiphosts table results in host = NULL + book getword chiphosts $1 $2 -var host + + if ("$host" == "NULL") + host anyhost + else + host $host + end +end + +macro get.host.for.camera + if ($0 != 3) + echo "USAGE: get.host.for.camera (camera) (class_id)" + break + end + + if (not($PARALLEL)) + echo local + return + end + + # missing camera and/or chiphosts table results in host = NULL + book getword chiphosts $1 $2 -var host + + if ("$host" == "NULL") + echo anyhost + else + echo $host + end +end + +macro check.globals + if ($?NETWORK == 0) + echo "NETWORK not defined: load pantasks.pro first" + break + end + if ($?PARALLEL == 0) + echo "PARALLEL not defined: load pantasks.pro first" + break + end + if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break + end + if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break + end + if ($?KEEP_FAILURES == 0) + echo "KEEP_FAILURES not defined: load pantasks.pro first" + break + end +end Index: /branches/eam_branches/eam_branch_20080806/ippTasks/pstamp.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/pstamp.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/pstamp.pro (revision 22160) @@ -0,0 +1,354 @@ + +#$VERBOSE = 3 + +# these variables wil cycle through the known database names + +$pstampDS_DB = 0 +$pstampReq_DB = 0 +$pstampJob_DB = 0 +$pstampFin_DB = 0 + +macro pstamp.reset + book init pstampRequest + book init pstampJob + book init pstampFinish +end + +pstamp.reset + + +macro pstamp.on + task pstamp.request.find + active true + end + task pstamp.request.load + active true + end + task pstamp.request.run + active true + end + + task request.finish.load + active true + end + task request.finish.run + active true + end + + task pstamp.job.load + active true + end + task pstamp.job.run + active true + end +end + +macro pstamp.off + task pstamp.request.find + active false + end + task pstamp.request.load + active false + end + task pstamp.request.run + active false + end + task request.finish.load + active false + end + task request.finish.run + active false + end + + task pstamp.job.load + active false + end + task pstamp.job.run + active false + end +end + +task pstamp.request.find + host local + + periods -poll $LOADPOLL + periods -exec 10 + periods -timeout 30 + npending 1 + + task.exec + if ($DB:n == 0) + option DEFAULT + command pstamp_queue_requests.pl --limit 5 + else + option $DB:$pstampDS_DB + command pstamp_queue_requests.pl --limit 5 --dbname $DB:$pstampDS_DB --verbose + $pstampDS_DB ++ + if ($pstampDS_DB >= $DB:n) set pstampDS_DB = 0 + end + end + + task.exit $EXIT_SUCCESS + # echo nothing to do + end + + task.exit default + showcommand failure + end + + task.exit timeout + showcommand timeout + end +end + +task pstamp.request.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 300 + npending 1 + + task.exec + if ($DB:n == 0) + option DEFAULT + command pstamptool -pendingreq + else + option $DB:$pstampReq_DB + command pstamptool -pendingreq -dbname $DB:$pstampReq_DB + $pstampReq_DB ++ + if ($pstampReq_DB >= $DB:n) set pstampReq_DB = 0 + end + end + + task.exit $EXIT_SUCCESS + ipptool2book stdout pstampRequest -key req_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + echo starting request + book listbook pstampRequest + end + + process_cleanup pstampRequest + end + + task.exit default + showcommand failure + end + + task.exit timeout + showcommand timeout + end +end + +task pstamp.request.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + npending 1 + + task.exec + book npages pstampRequest -var N + if ($N == 0) break + + book getpage pstampRequest 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pstampRequest $pageName pantaskState RUN + book getword pstampRequest $pageName req_id -var REQ_ID + book getword pstampRequest $pageName dbname -var DBNAME + + $run = pstamp_parser_run.pl --req_id $REQ_ID + + add_standard_args run + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + + task.exit default +# echo pstamp.request.run task.exit status: $JOB_STATUS + process_exit pstampRequest $options:0 $JOB_STATUS + end + + task.exit timeout + showcommand timeout + book setword pstampRequest $options:0 pantaskState TIMEOUT + end +end + +task request.finish.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 300 + npending 1 + + task.exec + if ($DB:n == 0) + option DEFAULT + command pstamptool -completedreq + else + option $DB:$pstampFin_DB + command pstamptool -completedreq -dbname $DB:$pstampFin_DB + $pstampFin_DB ++ + if ($pstampFin_DB >= $DB:n) set pstampFin_DB = 0 + end + end + + task.exit $EXIT_SUCCESS + ipptool2book stdout pstampFinish -key req_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook pstampFinish + end + + process_cleanup pstampFinish + end + + task.exit default + showcommand failure + end + + task.exit timeout + showcommand timeout + end +end + +task request.finish.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + npending 1 + + task.exec + book npages pstampFinish -var N + if ($N == 0) break + + book getpage pstampFinish 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pstampFinish $pageName pantaskState RUN + book getword pstampFinish $pageName req_id -var REQ_ID + book getword pstampFinish $pageName dbname -var DBNAME + book getword pstampFinish $pageName reqType -var REQ_TYPE + book getword pstampFinish $pageName name -var REQ_NAME + book getword pstampFinish $pageName outProduct -var PRODUCT + + $run = request_finish.pl --req_id $REQ_ID --req_type $REQ_TYPE --req_name $REQ_NAME --product $PRODUCT + + add_standard_args run + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + task.exit default + # echo request.finish.run task.exit status: $JOB_STATUS + process_exit pstampFinish $options:0 $JOB_STATUS + end + + task.exit timeout + showcommand timeout + book setword pstampFinish $options:0 pantaskState TIMEOUT.run + end +end + +task pstamp.job.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + if ($DB:n == 0) + option DEFAULT + command pstamptool -pendingjob -limit 10 + else + option $DB:$pstampJob_DB + command pstamptool -pendingjob -limit 10 -dbname $DB:$pstampJob_DB + $pstampJob_DB ++ + if ($pstampJob_DB >= $DB:n) set pstampJob_DB = 0 + end + end + + task.exit $EXIT_SUCCESS + # echo ipptool2book stdout pstampJob -key job_id -setword dbname $options:0 -setword pantaskState INIT + ipptool2book stdout pstampJob -key job_id -uniq -setword dbname $options:0 -setword pantaskState INIT + + book npages pstampJob -var N + if ($VERBOSE > 2) + book listbook pstampJob + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup pstampJob + end + + task.exit default + showcommand failure + end + + task.exit timeout + showcommand timeout + end +end + +task pstamp.job.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 1200 + npending 1 + + task.exec + book npages pstampJob -var N + if ($N == 0) break + + book getpage pstampJob 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + #echo pageName: $pageName + + book setword pstampJob $pageName pantaskState RUN + book getword pstampJob $pageName job_id -var JOB_ID + book getword pstampJob $pageName dbname -var DBNAME + + if ($VERBOSE > 1) + book listpage pstampJob $pageName + end + + + $run = pstamp_job_run.pl --job_id $JOB_ID + add_standard_args run + + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + + task.exit default + if ($VERBOSE > 1) + echo pstamp.job.run task.exit $JOB_ID status: $JOB_STATUS + end + process_exit pstampJob $options:0 $JOB_STATUS + end + + task.exit timeout + echo pstamp.job.run task.timeout $JOB_ID status: $JOB_STATUS + process_exit pstampJob $options:0 $JOB_STATUS + showcommand timeout + book setword pstampJob $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080806/ippTasks/register.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/register.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/register.pro (revision 22160) @@ -0,0 +1,304 @@ +## register.pro : tasks for image registration (insert into database) : -*- sh -*- +## this file contains the tasks for running the registration stage +## these tasks use the books regPendingImfile and regPendingExp + +# test for required global variables +check.globals + +book init regPendingImfile +book init regPendingExp + +macro register.reset + book init regPendingImfile + book init regPendingExp +end + +macro register.status + book listbook regPendingImfile + book listbook regPendingExp +end + +macro register.on + task register.imfile.load + active true + end + task register.imfile.run + active true + end + task register.exp.load + active true + end + task register.exp.run + active true + end +end + +macro register.off + task register.imfile.load + active false + end + task register.imfile.run + active false + end + task register.exp.load + active false + end + task register.exp.run + active false + end +end + +# these variables will cycle through the known database names +$regPendingImfile_DB = 0 +$regPendingExp_DB = 0 + +# select images ready for register analysis +# new entries are added to regPendingImfile +# compare the new list with the ones already selected +task register.imfile.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGDIR/register.imfile.load.log + + # select entries from the current DB; cycle to the next DB, if it exists + # iff the DB list is not set, use the value defined in .ipprc + task.exec + if ($DB:n == 0) + option DEFAULT + command regtool -pendingimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$regPendingImfile_DB + command regtool -pendingimfile -limit 20 -dbname $DB:$regPendingImfile_DB + $regPendingImfile_DB ++ + if ($regPendingImfile_DB >= $DB:n) set regPendingImfile_DB = 0 + end + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout regPendingImfile -key exp_id:tmp_class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + book shuffle regPendingImfile + if ($VERBOSE > 2) + book listbook regPendingImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup regPendingImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the register_imfile.pl script on pending images +task register.imfile.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + if ($NETWORK == 0) break + book npages regPendingImfile -var N + if ($N == 0) + periods -exec $RUNEXEC + break + end + periods -exec 0.05 + + # look for new images in regPendingImfile + book getpage regPendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword regPendingImfile $pageName pantaskState RUN + + book getword regPendingImfile $pageName exp_id -var EXP_ID + book getword regPendingImfile $pageName tmp_exp_name -var TMP_EXP_NAME + book getword regPendingImfile $pageName tmp_camera -var TMP_CAMERA + book getword regPendingImfile $pageName tmp_class_id -var TMP_CLASS_ID + book getword regPendingImfile $pageName uri -var URI + book getword regPendingImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword regPendingImfile $pageName dbname -var DBNAME + + # EXP_TAG is used to generate the unique, but human-readable, filenames + sprintf EXP_TAG "%s.%s" $TMP_EXP_NAME $EXP_ID + + # specify choice of remote host + set.host.for.camera $TMP_CAMERA $TMP_CLASS_ID + + # set the WORKDIR variable + set.workdir.by.camera $TMP_CAMERA $TMP_CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + ## generate outroot specific to this exposure (& chip) + sprintf logfile "%s/%s/%s.reg.%s.log" $WORKDIR $EXP_TAG $EXP_TAG $TMP_CLASS_ID + + stdout $LOGDIR/register.imfile.run.log + stderr $LOGDIR/register.imfile.run.log + + # XXX register_imfile.pl differs from the standard script : it does not have an 'outroot' argument, and it does not take '--redirect' + $run = register_imfile.pl --exp_id $EXP_ID --tmp_class_id $TMP_CLASS_ID --tmp_exp_name $TMP_EXP_NAME --uri $URI --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit regPendingImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword regPendingImfile $options:0 pantaskState TIMEOUT + end +end + +# select exposures ready for register_exp.pl +task register.exp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/register.exp.load.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command regtool -pendingexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$regPendingExp_DB + command regtool -pendingexp -limit 20 -dbname $DB:$regPendingExp_DB + $regPendingExp_DB ++ + if ($regPendingExp_DB >= $DB:n) set regPendingExp_DB = 0 + end + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout regPendingExp -key exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook regPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup regPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the register_exp.pl script on pending exposures +task register.exp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages regPendingExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in regPendingExp (pantaskState == INIT) + book getpage regPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword regPendingExp $pageName pantaskState RUN + book getword regPendingExp $pageName exp_id -var EXP_ID + book getword regPendingExp $pageName tmp_exp_name -var TMP_EXP_NAME + book getword regPendingExp $pageName tmp_camera -var TMP_CAMERA + book getword regPendingExp $pageName workdir -var WORKDIR_TEMPLATE + book getword regPendingExp $pageName dbname -var DBNAME + + # EXP_TAG is used to generate the unique, but human-readable, filenames + sprintf EXP_TAG "%s.%s" $TMP_EXP_NAME $EXP_ID + + # specify choice of remote host + set.host.for.camera $TMP_CAMERA $TMP_CLASS_ID + + # set the WORKDIR variable + set.workdir.by.camera $TMP_CAMERA $TMP_CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + ## generate output log based on filerule + sprintf logfile "%s/%s/%s.reg.log" $WORKDIR $EXP_TAG $EXP_TAG + + stdout $LOGDIR/register.exp.run.log + stderr $LOGDIR/register.exp.run.log + + $run = register_exp.pl --exp_id $EXP_ID --exp_tag $EXP_TAG --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit regPendingExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword regPendingExp $options:0 pantaskState TIMEOUT + end +end + +## XXX add a global path to output files Index: /branches/eam_branches/eam_branch_20080806/ippTasks/replicate.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/replicate.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/replicate.pro (revision 22160) @@ -0,0 +1,168 @@ +## replicate.pro : tasks for data replication : -*- sh -*- +## this file contains the tasks for maintaining duplicates as needed in Nebulous +## these tasks use the books replicatePending + +# test for required global variables +check.globals + +$NEB_DB = nebulous +$NEB_HOST = alala +$NEB_USER = jhipp +$NEB_PASS = jhipp + +$LOGSUBDIR = $LOGDIR/replicate +exec mkdir -p $LOGSUBDIR + +book init replicatePending + +macro replicate.reset + book init replicatePending +end + +macro replicate.status + book listbook replicatePending +end + +macro replicate.on + task replicate.load + active true + end + task replicate.run + active true + end +end + +macro replicate.off + task replicate.load + active false + end + task replicate.run + active false + end +end + +macro set.host.for.replicate + if ($0 != 2) + echo "USAGE: set.host.for.replicate (hostname)" + break + end + + if (not($PARALLEL)) + host local + return + end + +# parse volume name + + if ("$1" == "NULL") + host anyhost + else + host $1 + end +end + +# the replicate process interacts with only the single Nebulous server + +# select Nebulous objects which desire additional copies +task replicate.load + host local + + # modify these after the tasks are tested + periods -poll 10 + periods -exec 1 + periods -timeout 300 + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/replicate.log + + task.exec + # command does not need to be dynamic, but having it so allows us to adjust the periods + # so that we dont have to wait 10 minutes for things to start up + command neb-admin --host $NEB_HOST --db $NEB_DB --user $NEB_USER --pass $NEB_PASS --pendingreplicate --limit 2000 + periods -exec 600 + + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout replicatePending -key key -uniq -setword pantaskState INIT + + if ($VERBOSE > 2) + book listbook replicatePending + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup replicatePending + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# create the desired replicas +task replicate.run + periods -poll $RUNPOLL + periods -exec 30 + periods -timeout 30 + + task.exec + book npages replicatePending -var N + if ($NETWORK == 0) break + if ($N == 0) + periods -exec 30 + break + end + periods -exec 0.05 + + # look for new objects in replicatePending + book getpage replicatePending 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword replicatePending $pageName pantaskState RUN + + # XXX what values do I need to get back? + book getword replicatePending $pageName key -var KEY + book getword replicatePending $pageName need_copies -var NEED_COPIES + book getword replicatePending $pageName volume_name -var VOLUME_NAME + book getword replicatePending $pageName volume_host -var VOLUME_HOST + + set.host.for.replicate $VOLUME_HOST + + stdout NULL + stderr $LOGSUBDIR/replicate.log + + # these operations do not require a database to be specified + $run = neb-replicate --copies $NEED_COPIES $KEY + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit replicatePending $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword replicatePending $options:0 pantaskState TIMEOUT + end +end + Index: /branches/eam_branches/eam_branch_20080806/ippTasks/simtest.auto =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/simtest.auto (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/simtest.auto (revision 22160) @@ -0,0 +1,119 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis + +automate METADATA + name STR BIAS + check STR "regtool -processedexp -exp_type BIAS -inst SIMTEST -dbname @DBNAME@" + ncheck S32 20 + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst SIMTEST -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@" +END + +automate METADATA + name STR DARK + check STR "detselect -search -inst SIMTEST -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst SIMTEST -det_type DARK -select_exp_type DARK -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -dbname @DBNAME@" +END + +automate METADATA + name STR SHUTTER + check STR "detselect -search -inst SIMTEST -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst SIMTEST -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-r + check STR "detselect -search -inst SIMTEST -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst SIMTEST -det_type FLAT -filter r -select_exp_type FLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-i + check STR "detselect -search -inst SIMTEST -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst SIMTEST -det_type FLAT -filter i -select_exp_type FLAT -select_filter i -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter i -dbname @DBNAME@" +END + +automate METADATA + name STR OBJECT-i + check STR "detselect -dbname @DBNAME@ -search -inst SIMTEST -filter i -det_type FLAT" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst SIMTEST -filter i -label wait -set_label proc" + block STR NONE +END + +automate METADATA + name STR OBJECT-r + check STR "detselect -dbname @DBNAME@ -search -inst SIMTEST -filter r -det_type FLAT" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst SIMTEST -filter r -label wait -set_label proc" + block STR NONE +END + +### Stack automation??? +automate METADATA + name STR STACK + regular STR "stacktool -definebyquery -all -workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@" +END + +### Diff automation??? +automate METADATA + name STR DIFF + regular STR "difftool -definebyquery -workdir file://@CWD@/diff -good_frac 0.2 -dbname @DBNAME@" +END + +### Magic automation??? +automate METADATA + name STR MAGIC + regular STR "magictool -definebyquery -workdir file://@CWD@/magic -good_frac 0.2 -dbname @DBNAME@" +END + + + + +### automate METADATA +### name STR OBJECT-r +### check STR "detselect -search -inst SIMTEST -det_type FLAT -dbname @DBNAME@ -filter r" +### launch STR "chiptool -unblock -label simtest-r -dbname @DBNAME@" +### block STR "chiptool -unmasked -label simtest-r -dbname @DBNAME@" +### END + +### automate METADATA +### name STR OBJECT-i +### check STR "detselect -search -inst SIMTEST -det_type FLAT -dbname @DBNAME@ -filter i" +### launch STR "chiptool -unblock -label simtest-i -dbname @DBNAME@" +### block STR "chiptool -unmasked -label simtest-i -dbname @DBNAME@" +### END + +### there is a weakness in the label / block business: the labels are +### generic, and the blocks are against those fairly generic words. +### that makes it difficult to block and unblock science exposures +### based on different detrend types that are available. For example, +### to block the r-band against the absence of the desired r-band +### flat, we would need to specify a label specific to the r-band +### images and remove that block when ready. + +### a better approach might be to modify the labels rather than the +### blocks. we can define, eg, chiptool -set_label -definebyquery to +### turn assign the label names based on various properties of the +### interesting images. + +### here is what the automate element might look like for such a circumstance: + +### automate METADATA +### name STR OBJECT-i +### block STR "chiptool -dbname @DBNAME@ -unmasked -label wait -filter i -time_begin 2008/1/1 -time_end 2008/1/2" +### check STR "detselect -dbname @DBNAME@ -search -inst SIMTEST -det_type FLAT -dbname @DBNAME@ -filter i -time_begin 2008/1/1 -time_end 2008/1/2" +### launch STR "chiptool -dbname @DBNAME@ -set_label proc -label wait -dbname @DBNAME@ -time_begin 2008/1/1 -time_end 2008/1/2" +### END + +### XXX still not quite there.... +### the 'checks' or maybe the 'launch' needs to include a '-n_exp_min' ? +### Index: /branches/eam_branches/eam_branch_20080806/ippTasks/simtest.basic.config =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/simtest.basic.config (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/simtest.basic.config (revision 22160) @@ -0,0 +1,112 @@ + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR SIMTEST + NIMAGES S32 20 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR SIMTEST + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 10, 10 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,i + @EXPTIMES F32 0.1,20.0,20.0 + + NSETUP S32 5 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 3 + DITHER.ND S32 2 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,i + @EXPTIMES F32 10.00,10.00 + @SKYMAGS F32 20.86,20.15 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + + +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 1 + DITHER.ND S32 1 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,r,i,i + @EXPTIMES F32 5.0, 240.0, 5.0, 240.0 + @SKYMAGS F32 20.86,20.86,20.15,20.15 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + Index: /branches/eam_branches/eam_branch_20080806/ippTasks/simtest.det =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/simtest.det (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/simtest.det (revision 22160) @@ -0,0 +1,42 @@ + +detRunDef MULTI + +detRunDef METADATA + detRunName STR BIAS + prereq STR NONE + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type BIAS -select_exp_type BIAS" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR DARK + prereq STR BIAS + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type DARK -select_exp_type DARK" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR SHUTTER + prereq STR DARK + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-r + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-i + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter i -select_exp_type FLAT -select_filter i" + dbname STR eamtest + pantaskState STR NULL +END Index: /branches/eam_branches/eam_branch_20080806/ippTasks/simtest.megacam.config =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/simtest.megacam.config (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/simtest.megacam.config (revision 22160) @@ -0,0 +1,67 @@ + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR MEGACAM + + BIAS.LEVEL F32 100.0 + BIAS.RANGE F32 10.0 + + NIMAGE S32 15 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR MEGACAM + + DARK.MIN F32 100.0 + DARK.MAX F32 200.0 + + NIMAGE S32 15 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR MEGACAM + + FILTERS STR g,r,i + @EXPTIMES F32 60.0,30.0,30.0 + + NSETUP S32 15 +END + +SEQUENCE_SKIP METADATA + OBSTYPE STR OBJECT + CAMERA STR MEGACAM + + CENTER.RA F32 10.0 + CENTER.DEC F32 60.0 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 5 + OFFSET.ND S32 5 + + DITHER.RA F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 2 + DITHER.ND S32 2 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.grizy + FILTERS STR g,r,i + @EXPTIMES F32 60.0,30.0,30.0 + @SKYMAGS F32 21.0,20.5,20.0 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.5 + IQ_MAX F32 0.8 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + Index: /branches/eam_branches/eam_branch_20080806/ippTasks/simtest.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/simtest.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/simtest.pro (revision 22160) @@ -0,0 +1,50 @@ +## simtest.pro : automatic full-suite simulated IPP test : -*- sh -*- + +macro simtest + if ($0 != 5) + echo "USAGE: simtest (testroot) (dbname) (hostname) (init)" + echo " if (init) == 0, just run the analysis tasks" + echo " if (init) == 1, (re)create the database and (re)inject the images" + echo " if (init) == 2, restart completely from scratch" + break + end + + $testroot = $1 + $dbname = $2 + $hostname = $3 + + if ($4 > 0) + # XXX this will fail and exit the script if the db does not exist or is old... + exec pxadmin -delete -dbname $dbname + + # XXX this gives warnings if the db exists... + exec pxadmin -create -dbname $dbname + + if ($4 == 2) + # the labels "wait" and "proc" are special names used in automate.pro + exec ppSimSequence $MODULES:0/$testroot.config $testroot.mkimages $testroot.inject -path raw -workdir work -dbname $dbname -label wait -dvodb DVO -tess_id TESS + exec source $testroot.mkimages + + file TESS/Images.dat found + if ($found == 0) + exec skycells 8 -scale 0.2 -D CATDIR TESS + end + end + + exec chiptool -dbname $dbname -block -label wait + exec source $testroot.inject + end + + module pantasks.pro + module automate.pro + + module.tasks + + controller host add $hostname + add.database $dbname + + automate.load simtest.auto $dbname + run +end + +# XXX add a test function to pxadmin to check for db existence Index: /branches/eam_branches/eam_branch_20080806/ippTasks/site.manoa.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/site.manoa.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/site.manoa.pro (revision 22160) @@ -0,0 +1,52 @@ +## site.mhpcc.pro : example configuration script for the MHPCC IPP cluster : -*- sh -*- + +## a site configuration needs to define the following pieces of information: +## 1) the machines used for processing +## 2) the default_host and workdir_template variables +## 3) the chiphost table + +macro init.cluster.po + $PARALLEL = 1 + controller exit true + + # po05 -- using for pantasks + # po23 -- broken perl + + controller host add po02 + controller host add po03 + controller host add po04 +# controller host add po05 + controller host add po06 + controller host add po07 +# controller host add po08 +# controller host add po09 +# controller host add po10 +# controller host add po11 +# controller host add po12 +# controller host add po13 +# controller host add po14 +# controller host add po15 +# controller host add po16 +# controller host add po17 +# controller host add po18 +# controller host add po19 +# controller host add po20 +# controller host add po21 +# controller host add po22 +# controller host add po23 +# controller host add po24 +end + +macro init.cluster.sn + $PARALLEL = 1 + controller exit true + controller host add sn2 + controller host add sn3 + controller host add sn4 + controller host add sn5 +end + +macro init.site + queueload tmp -x "cat $MODULES:0/chiphosts.manoa.config" + ipptool2book tmp chiphosts -key camera +end Index: /branches/eam_branches/eam_branch_20080806/ippTasks/site.mhpcc.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/site.mhpcc.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/site.mhpcc.pro (revision 22160) @@ -0,0 +1,68 @@ +## site.mhpcc.pro : example configuration script for the MHPCC IPP cluster : -*- sh -*- + +## a site configuration needs to define the following pieces of information: +## 1) the machines used for processing +## 2) the default_host and workdir_template variables +## 3) the chiphost table + +macro init.cluster.mhpcc + $PARALLEL = 1 + controller exit true + controller host add ipp005 + controller host add ipp006 + controller host add ipp007 + controller host add ipp008 + controller host add ipp009 + controller host add ipp010 + controller host add ipp011 + controller host add ipp016 + controller host add ipp020 + controller host add ipp021 +end + +## override the basic inits set in pantasks.pro +macro init.copy.mhpcc + if ($0 != 2) + echo "USAGE: init.copy.mhpcc (nebulous)" + echo "nebulous may be 'on' or 'off'" + break + end + + if (("$1" != "on") && ("$1" != "off")) + echo "USAGE: init.copy.mhpcc (nebulous)" + echo "nebulous may be 'on' or 'off'" + break + end + + # XXX this is only used by summit.copy.pro. move this as a check into summit.copy.pro? + $COMPRESS = 1 + + # the templates are used if we have a class_id/host relationship; + # if none is found, the default values are used + # XXX not sure how to handle the .N value if we need to use more than one + + if ("$1" == "on") + $NEBULOUS = 1 + $default_host = ipp004 + $workdir_template = neb://@HOST@.0 + else + $NEBULOUS = 0 + $default_host = ipp004 + $workdir_template = /data/@HOST@.0 + end +end + +macro init.site + init.cluster.mhpcc + init.copy.mhpcc on + + queueload tmp -x "cat $MODULES:0/chiphosts.mhpcc.config" + ipptool2book tmp chiphosts -key camera +end + +macro init.site.nohosts + init.copy.mhpcc on + + queueload tmp -x "cat $MODULES:0/chiphosts.mhpcc.config" + ipptool2book tmp chiphosts -key camera +end Index: /branches/eam_branches/eam_branch_20080806/ippTasks/stack.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/stack.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/stack.pro (revision 22160) @@ -0,0 +1,287 @@ +## stack.pro : tasks for image stacking : -*- sh -*- + +## This file contains panTasks definitions for performing the image stacking. +## After a stack (with associated stack_id) is defined, the stack is performed +## (tasks in stackSumSkyfile). + +# test for required global variables +check.globals + +### Initialise the books containing the tasks to do +book init stackSumSkyfile +book init stackCleanup + +### Database lists +$stackSkycell_DB = 0 +$stackCleanup_DB = 0 + +### Check status of stacking tasks +macro stack.status + book listbook stackSumSkyfile + book listbook stackCleanup +end + +### Reset stacking tasks +macro stack.reset + book init stackSumSkyfile + book init stackCleanup +end + +### Turn stacking tasks on +macro stack.on + task stack.skycell.load + active true + end + task stack.skycell.run + active true + end + task stack.cleanup.load + active true + end + task stack.cleanup.run + active true + end +end + +### Turn stacking tasks off +macro stack.off + task stack.skycell.load + active false + end + task stack.skycell.run + active false + end + task stack.cleanup.load + active false + end + task stack.cleanup.run + active false + end +end + + +### Load tasks for doing the stack +### Tasks are loaded into stackSumSkyfile. +task stack.skycell.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/stack.skycell.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command stacktool -tosum -limit 20 + else + # save the DB name for the exit tasks + option $DB:$stackSkycell_DB + command stacktool -tosum -limit 20 -dbname $DB:$stackSkycell_DB + $stackSkycell_DB ++ + if ($stackSkycell_DB >= $DB:n) set stackSkycell_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout stackSumSkyfile -key stack_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook stackSumSkyfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup stackSumSkyfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the stack overlaps +### Tasks are taken from stackSumSkyfile. +task stack.skycell.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + book npages stackSumSkyfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in stackSumSkyfile (pantaskState == INIT) + book getpage stackSumSkyfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword stackSumSkyfile $pageName pantaskState RUN + book getword stackSumSkyfile $pageName stack_id -var STACK_ID + book getword stackSumSkyfile $pageName tess_id -var TESS_DIR + book getword stackSumSkyfile $pageName skycell_id -var SKYCELL_ID + book getword stackSumSkyfile $pageName camera -var CAMERA + book getword stackSumSkyfile $pageName workdir -var WORKDIR_TEMPLATE + book getword stackSumSkyfile $pageName dbname -var DBNAME + + ### XXX Instead of setting these by camera, need to set by skycell_id +# set.host.for.camera $CAMERA $STACK_ID +# set.workdir.by.camera $CAMERA $STACK_ID $WORKDIR_TEMPLATE $default_host WORKDIR + host anyhost + $WORKDIR = $WORKDIR_TEMPLATE + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.stk.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $STACK_ID + + ## generate output log based on filerule (convert the URI to a PATH) (use STACK_ID?) +# $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $STACK_ID --basename $outroot` + $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id NONE --basename $outroot` + if ("$logfile" == "") + echo "WARNING: logfile not defined in stack.pro:135" + break + end + + stdout $logfile + stderr $logfile + dirname $logfile -var outpath + mkdir $outpath + + $run = stack_skycell.pl --stack_id $STACK_ID --outroot $outroot + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit stackSumSkyfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword stackSumSkyfile $options:0 pantaskState TIMEOUT + end +end + + +# select images ready for stack analysis +# new entries are added to stackCleanup +# skip already-present entries +task stack.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/stack.cleanup.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command stacktool -pendingcleanuprun -limit 20 + else + # save the DB name for the exit tasks + option $DB:$stackCleanup_DB + command stacktool -pendingcleanuprun -limit 20 -dbname $DB:$stackCleanup_DB + $stackCleanup_DB ++ + if ($stackCleanup_DB >= $DB:n) set stackCleanup_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout stackCleanup -key stack_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook stackCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup stackCleanup + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task stack.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages stackCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in stackCleanup (pantaskState == INIT) + book getpage stackCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword stackCleanup $pageName pantaskState RUN + book getword stackCleanup $pageName camera -var CAMERA + book getword stackCleanup $pageName state -var CLEANUP_MODE + book getword stackCleanup $pageName stack_id -var STACK_ID + book getword stackCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and stack (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/stack.cleanup.log + stderr $LOGDIR/stack.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage stack --stage_id $STACK_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit stackCleanup $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword stackCleanup $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080806/ippTasks/summit.copy.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/summit.copy.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/summit.copy.pro (revision 22160) @@ -0,0 +1,434 @@ +## summit.copy.pro : tasks for the summit to IPP download : -*- sh -*- +## PanTasks scripts for Summit Copy + +## XXX note that this currently works with a single database as defined in .ipprc +## XXX tie the database for output to the database from which the datastore was determined + +# pztool -adddatastore -inst isp -telescope ps1 -uri http://otis1.ifa.hawaii.edu/ds/skyprobe/index.txt +# pztool -adddatastore -inst gpc1 -telescope ps1 -uri http://conductor/ds/gpc1/index.txt +# pztool -adddatastore -inst allskycam -telescope ps1 -uri http://otis1.ifa.hawaii.edu/ds/allskycam/index.txt + +# NOTE: workdir / volume mangling and nebulous. these tasks copy the +# imfiles from the summit, placing the output files in a directory +# which is based on the chip/host relationship. If nebulous is being +# used, the volume name is set based on the host; if nebulous is not +# being used, the workdir is set to include the host name. The copy +# operation is targetted to the same host by pcontrol. The value of +# workdir is set to be a template into which the appropriate value of +# @HOST@ may be substituted in later scripts. + +# test for required global variables +check.globals + +# list of DataStores to pull data from +book init pzDataStore +# list of summit exps that need to be queried +book init pzPendingExp +# list of summit imfiles that need to be downloaded +book init pzPendingImfile + +macro copy.on + task pztool.datastore + active true + end + task pzgetexp + active true + end + task pztool.pendingexp + active true + end + task pzgetimfile + active true + end + task pztool.pendingimfile + active true + end + task summit_copy + active true + end +end + +macro copy.off + task pztool.datastore + active false + end + task pzgetexp + active false + end + task pztool.pendingexp + active false + end + task pzgetimfile + active false + end + task pztool.pendingimfile + active false + end + task summit_copy + active false + end +end + +# these variables will cycle through the known database names +$pztoolDatastore_DB = 0 +$pztoolPendingExp_DB = 0 +$pztoolPendingImfile_DB = 0 + +# build a book of datastores to poll for data +task pztool.datastore + host local + + # timeout shorter than exec so jobs do not build up + periods -exec 10 + periods -poll 1 + periods -timeout 20 + npending 1 + # trange 16:00 23:59 + # trange 00:00 04:00 + + task.exec + if ($DB:n == 0) + option DEFAULT + command pztool -datastore + else + # save the DB name for the exit tasks + option $DB:$pztoolDatastore_DB + command pztool -datastore -dbname $DB:$pztoolDatastore_DB + $pztoolDatastore_DB ++ + if ($pztoolDatastore_DB >= $DB:n) set pztoolDatastore_DB = 0 + end + end + + # success + task.exit 0 + # flush pzDataStore book + book init pzDataStore + # convert 'stdout' to book format + ipptool2book stdout pzDataStore -key camera:telescope -uniq -setword dbname $options:0 + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout + end +end + +$datastore_index = 0 + +# run pzgetexp periodically to populate pzPendingExp in the database (no I/O) +# this task is querying the data store for a list of exposures ("filesets") +# and inserting these into a db table on the local cluster (pzPendingExp) +task pzgetexp + periods -exec 60 + periods -poll 1 + periods -timeout 700 + # trage 16:00 23:59 + # trage 00:00 04:00 + npending 1 + host local + + task.exec + # find an exp that needs imfiles fetched + book getpage pzDataStore $datastore_index -var pageName + if ("$pageName" == "NULL") break + + # increment our pzDataStore index and loop back to 0 and the end of the + # book + $datastore_index ++ + book npages pzDataStore -var npages + if ($datastore_index == $npages ) + $datastore_index = 0 + end + + book getword pzDataStore $pageName camera -var CAMERA + book getword pzDataStore $pageName telescope -var TELESCOPE + book getword pzDataStore $pageName uri -var URI + book getword pzDataStore $pageName dbname -var DBNAME + + # store the current page + options $pageName + + $run = pzgetexp -uri $URI -inst $CAMERA -telescope $TELESCOPE -dbname $DBNAME -timeout 650 + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + + command $run + end + + task.exit 0 + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout + end +end + +# build a book of exps/filesetids that need to be queried +task pztool.pendingexp + host local + + periods -exec 10 + periods -poll 1 + periods -timeout 20 + # trange 16:00 23:59 + # trange 00:00 04:00 + npending 1 + + task.exec + if ($DB:n == 0) + option DEFAULT + command pztool -pendingexp -limit 5 + else + # save the DB name for the exit tasks + option $DB:$pztoolPendingExp_DB + command pztool -pendingexp -limit 5 -dbname $DB:$pztoolPendingExp_DB + $pztoolPendingExp_DB ++ + if ($pztoolPendingExp_DB >= $DB:n) set pztoolPendingExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout pzPendingExp -key exp_name:camera:telescope -uniq -setword dbname $options:0 -setword pantaskState INIT + + # delete existing entries in the appropriate pantaskStates + process_cleanup pzPendingExp + end + + task.exit default + showcommand failure + if ($VERBOSE) + echo "*** stdout ***" + queueprint stdout + echo "*** stderr ***" + queueprint stderr + end + end + task.exit timeout + showcommand timeout + end +end + +# run pzgetimfiles on pending exps. analogous to pzgetexp, this task +# is downloading a list of imfiles reported by the datastore for a +# given exposure ("fileset"), and placing the result in the local +# database table of imfiles +task pzgetimfile + periods -exec 0.05 + periods -poll 0.025 + periods -timeout 700 + # trage 16:00 23:59 + # trage 00:00 04:00 + host local + npending 10 + + task.exec + if ($NETWORK == 0) break + + # if we are waiting on data, make the interval long + book npages pzPendingExp -var N + if ($N == 0) + periods -exec 20 + break + end + periods -exec 0.05 + + # find an exp that needs imfiles fetched + book getpage pzPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # set that exp to run + book setword pzPendingExp $pageName pantaskState RUN + + book getword pzPendingExp $pageName exp_name -var EXP_NAME + book getword pzPendingExp $pageName camera -var CAMERA + book getword pzPendingExp $pageName telescope -var TELESCOPE + book getword pzPendingExp $pageName dateobs -var DATEOBS + book getword pzPendingExp $pageName exp_type -var EXP_TYPE + book getword pzPendingExp $pageName uri -var URI + book getword pzPendingExp $pageName imfiles -var IMFILES + book getword pzPendingExp $pageName dbname -var DBNAME + + # store the current page + options $pageName + + $batman = $EXP_NAME + # Sidik says we should use a longer timeout + $run = pzgetimfiles -uri $URI -filesetid $batman -inst $CAMERA -telescope $TELESCOPE -dbname $DBNAME -timeout 650 + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit 0 + process_exit pzPendingExp $options:0 $JOB_STATUS + end + + task.exit default + showcommand failure + process_exit pzPendingExp $options:0 $JOB_STATUS + end + + task.exit timeout + showcommand timeout + book setword pzPendingExp $options:0 pantaskState TIMEOUT + end +end + + +# build a book of imfiles/files that need to be downloaded +task pztool.pendingimfile + host local + + periods -exec 30 + periods -poll 1 + periods -timeout 120 + # trage 16:00 23:59 + # trage 00:00 04:00 + npending 1 + + # select entries from the current DB; cycle to the next DB, if it exists + # iff the DB list is not set, use the value defined in .ipprc + task.exec + if ($DB:n == 0) + option DEFAULT + command pztool -pendingimfile -limit 40 + else + # save the DB name for the exit tasks + option $DB:$pztoolPendingImfile_DB + command pztool -pendingimfile -limit 60 -dbname $DB:$pztoolPendingImfile_DB + $pztoolPendingImfile_DB ++ + if ($pztoolPendingImfile_DB >= $DB:n) set pztoolPendingImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout pzPendingImfile -key exp_name:camera:telescope:class:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + book shuffle pzPendingImfile + + # delete existing entries in the appropriate pantaskStates + process_cleanup pzPendingImfile + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout + end +end + +# retreive an imfile with dsget and then call pztool -copydone +task summit_copy + periods -exec 5 + periods -poll 0.05 + periods -timeout 650 + # trage 16:00 23:59 + # trage 00:00 04:00 + + task.exec + if ($NETWORK == 0) break + + # if we are waiting on data, make the interval long + book npages pzPendingImfile -var N + if ($N == 0) + periods -exec 20 + break + end + periods -exec 0.05 + + # find an exp that needs imfiles fetched + book getpage pzPendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # set that exp to run + book setword pzPendingImfile $pageName pantaskState RUN + + book getword pzPendingImfile $pageName uri -var URI + book getword pzPendingImfile $pageName bytes -var BYTES + book getword pzPendingImfile $pageName md5sum -var MD5SUM + book getword pzPendingImfile $pageName dateobs -var DATEOBS + book getword pzPendingImfile $pageName exp_name -var EXP_NAME + book getword pzPendingImfile $pageName camera -var CAMERA + book getword pzPendingImfile $pageName telescope -var TELESCOPE + book getword pzPendingImfile $pageName class -var CLASS + book getword pzPendingImfile $pageName class_id -var CLASS_ID + book getword pzPendingImfile $pageName dbname -var DBNAME + + set.host.for.camera $CAMERA $CLASS_ID + + # 2007-08-30T05:09:59Z + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 5 2 MONTH + substr $DATEOBS 8 2 DAY + + # we need to set the workdir based on 1) nebulous or not? 2) chip/host relationship + # this function uses workdir_template, default_host, volume_template, volume_default, + # it sets workdir and volume + set.workdir.by.camera $CAMERA $CLASS_ID $workdir_template $default_host workdir_base + + # figure out filename + # XXX may need to use sprintf here + $FILENAME = $workdir_base/$CAMERA/$YEAR\$MONTH\$DAY/$EXP_NAME/$EXP_NAME.$CLASS_ID.fits + $workdir = $workdir_template/$CAMERA/$YEAR\$MONTH\$DAY + + # workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb://@HOST@.0/gpc1/20080130 + + # filename examples: + # file://data/ipp005.0/gpc1/20080130/o4437g0025d/o4437g0025d.XY05.fits + # neb://@HOST@.0/gpc1/20080130/o4437g0025d/o4437g0025d.XY05.fits + + book setword pzPendingImfile $pageName filename $FILENAME + + $run = summit_copy.pl --uri $URI --filename $FILENAME --exp_name $EXP_NAME --inst $CAMERA --telescope $TELESCOPE --class $CLASS --class_id $CLASS_ID --bytes $BYTES --md5 $MD5SUM --end_stage reg --workdir $workdir --dbname $DBNAME --timeout 120 --verbose + if ($COMPRESS) + $run = $run --compress + else + $run = $run --bytes $BYTES + end + if (("$MD5SUM" != "NULL") && ("$MD5SUM" != "0") && (not($COMPRESS))) + $run = $run --md5 $MD5SUM + end + if ($NEBULOUS) + $run = $run --nebulous + end + # add_standard_args run + + # store the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit pzPendingImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword pzPendingImfile $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branches/eam_branch_20080806/ippTasks/warp.pro =================================================================== --- /branches/eam_branches/eam_branch_20080806/ippTasks/warp.pro (revision 22160) +++ /branches/eam_branches/eam_branch_20080806/ippTasks/warp.pro (revision 22160) @@ -0,0 +1,404 @@ +## warp.pro : image warping tasks : -*- sh -*- +## This file contains panTasks definitions for performing the image warping. + +### This is done in two (main) steps. After a warp (with associated warp_id) is defined, +### overlaps between the exposure being warped and skycells are calculated (tasks in warpInputExp). +### Then for each skycell, the warp is made (tasks in warpPendingSkycell). + +### Setups +check.globals + +### Initialise the books containing the tasks to do +book init warpInputExp +book init warpPendingSkyCell +book init warpPendingCleanup + +### Database lists +$warpExp_DB = 0 +$warpSkycell_DB = 0 +$warpCleanup_DB = 0 + +### Check status of warping tasks +macro warp.status + book listbook warpInputExp + book listbook warpPendingSkyCell + book listbook warpPendingCleanup +end + +### Reset warping tasks +macro warp.reset + book init warpInputExp + book init warpPendingSkyCell + book init warpPendingCleanup +end + +### Turn warping tasks on +macro warp.on + task warp.exp.load + active true + end + task warp.exp.run + active true + end + task warp.skycell.load + active true + end + task warp.skycell.run + active true + end + task warp.cleanup.load + active true + end + task warp.cleanup.run + active true + end +end + +### Turn warping tasks off +macro warp.off + task warp.exp.load + active false + end + task warp.exp.run + active false + end + task warp.skycell.load + active false + end + task warp.skycell.run + active false + end + task warp.cleanup.load + active false + end + task warp.cleanup.run + active false + end +end + +### Load tasks for calculating the warp overlaps +### Tasks are loaded into warpInputExp. +task warp.exp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/warp.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command warptool -tooverlap -limit 20 + else + # save the DB name for the exit tasks + option $DB:$warpExp_DB + command warptool -tooverlap -limit 20 -dbname $DB:$warpExp_DB + $warpExp_DB ++ + if ($warpExp_DB >= $DB:n) set warpExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout warpInputExp -key warp_id:fake_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpInputExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpInputExp + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the warp overlaps +### Tasks are taken from warpInputExp. +task warp.exp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages warpInputExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in warpInputExp (pantaskState == INIT) + book getpage warpInputExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpInputExp $pageName pantaskState RUN + book getword warpInputExp $pageName warp_id -var WARP_ID + book getword warpInputExp $pageName camera -var CAMERA + book getword warpInputExp $pageName workdir -var WORKDIR_TEMPLATE + book getword warpInputExp $pageName dbname -var DBNAME + # XXX change tess_id to tess_dir when schema is changed + book getword warpInputExp $pageName tess_id -var TESS_DIR + book getword warpInputExp $pageName exp_tag -var EXP_TAG + + ### XXX Instead of setting these by camera, need to set by skycell_id + set.host.for.camera $CAMERA $WARP_ID + set.workdir.by.camera $CAMERA $WARP_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure + sprintf logfile "%s/%s/%s.wrp.%s.log" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID + + stdout $LOGDIR/warp.exp.log + stderr $LOGDIR/warp.exp.log + + # XXX warp_overlap.pl differs from the standard script : it does not have an 'outroot' argument, and it does not take '--redirect' + $run = warp_overlap.pl --warp_id $WARP_ID --camera $CAMERA --tess_dir $TESS_DIR --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit warpInputExp $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpInputExp $options:0 pantaskState TIMEOUT + end +end + + +### Load tasks for doing the warps +### Tasks are loaded into warpPendingSkyCell. +task warp.skycell.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/warp.skycell.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command warptool -towarped -limit 20 + else + # save the DB name for the exit tasks + option $DB:$warpSkycell_DB + command warptool -towarped -limit 20 -dbname $DB:$warpSkycell_DB + $warpSkycell_DB ++ + if ($warpSkycell_DB >= $DB:n) set warpSkycell_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + # XXX change tess_id to tess_dir when db is updated + ipptool2book stdout warpPendingSkyCell -key warp_id:skycell_id:tess_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpPendingSkyCell + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpPendingSkyCell + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the warp overlaps +### Tasks are taken from warpPendingSkyCell. +task warp.skycell.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages warpPendingSkyCell -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in warpPendingSkyCell (pantaskState == INIT) + book getpage warpPendingSkyCell 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpPendingSkyCell $pageName pantaskState RUN + book getword warpPendingSkyCell $pageName warp_id -var WARP_ID + book getword warpPendingSkyCell $pageName skycell_id -var SKYCELL_ID + # XXX change tess_id to tess_dir when schema is changed + book getword warpPendingSkyCell $pageName tess_id -var TESS_DIR + book getword warpPendingSkyCell $pageName camera -var CAMERA + book getword warpPendingSkyCell $pageName workdir -var WORKDIR_TEMPLATE + book getword warpPendingSkyCell $pageName dbname -var DBNAME + + ### XXX Instead of setting these by camera, need to set by skycell_id + set.host.for.camera $CAMERA $WARP_ID + set.workdir.by.camera $CAMERA $WARP_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + # XXX we are supplied TESS_DIR, strip off path to generate TESS_ID + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.wrp.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $WARP_ID + + stdout $LOGDIR/warp.skycell.log + stderr $LOGDIR/warp.skycell.log + + $run = warp_skycell.pl --warp_id $WARP_ID --skycell_id $SKYCELL_ID --tess_dir $TESS_DIR --camera $CAMERA --outroot $outroot --redirect-output + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit warpPendingSkyCell $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpPendingSkyCell $options:0 pantaskState TIMEOUT + end +end + +# select images ready for warp analysis +# new entries are added to warpPendingImfile +# skip already-present entries +task warp.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/warp.cleanup.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command warptool -pendingcleanuprun -limit 20 + else + # save the DB name for the exit tasks + option $DB:$warpCleanup_DB + command warptool -pendingcleanuprun -limit 20 -dbname $DB:$warpCleanup_DB + $warpCleanup_DB ++ + if ($warpCleanup_DB >= $DB:n) set warpCleanup_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout warpPendingCleanup -key warp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpPendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpPendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task warp.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages warpPendingCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in warpPendingCleanup (pantaskState == INIT) + book getpage warpPendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpPendingCleanup $pageName pantaskState RUN + book getword warpPendingCleanup $pageName camera -var CAMERA + book getword warpPendingCleanup $pageName state -var CLEANUP_MODE + book getword warpPendingCleanup $pageName warp_id -var WARP_ID + book getword warpPendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and warp (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/warp.cleanup.log + stderr $LOGDIR/warp.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit warpPendingCleanup $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpPendingCleanup $options:0 pantaskState TIMEOUT + end +end