Index: /branches/jh_branches/backtrack/dbconfig/.cvsignore =================================================================== --- /branches/jh_branches/backtrack/dbconfig/.cvsignore (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/.cvsignore (revision 22250) @@ -0,0 +1,2 @@ +ippdb.m4 +ippdb.mdc Index: /branches/jh_branches/backtrack/dbconfig/Makefile =================================================================== --- /branches/jh_branches/backtrack/dbconfig/Makefile (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/Makefile (revision 22250) @@ -0,0 +1,26 @@ +SHELL=/bin/sh +GLUEFORGE=`which glueforge` + +all: ippdb.mdc + +ippdb.mdc : ipp.m4 *.md + m4 ipp.m4 > ippdb.mdc + +## please leave the output target as ippdb.src and +## move to ippdb by hand for cvs import +install: ippdb.mdc + rm -rf ../ippdb.src + $(GLUEFORGE) -i ippdb.mdc --output ../ippdb.src + chmod +x ../ippdb.src/autogen.sh + +src: ippdb.mdc + $(GLUEFORGE) -i ippdb.mdc --output ../ippdb + chmod +x ../ippdb/autogen.sh + +foo: src + $(MAKE) -C ../ippdb install + +build : ippdb ippdb.mdc + +clean: + @rm -f *~ Index: /branches/jh_branches/backtrack/dbconfig/cam.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/cam.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/cam.md (revision 22250) @@ -0,0 +1,37 @@ +camRun METADATA + cam_id S64 0 # Primary Key AUTO_INCREMENT + chip_id S64 0 # Primary Key fkey(chip_id) ref chipRun(chip_id) + state STR 64 # key + workdir STR 255 + workdir_state STR 64 # key + label STR 64 # key + reduction STR 64 + expgroup STR 64 # key + dvodb STR 255 +END + +camProcessedExp METADATA +# the camPendingExp row gets deleted so we can not put a fkey on cam_id + cam_id S64 0 # Primary Key + chip_id S64 0 # Primary Key fkey(cam_id, chip_id) ref camRun(cam_id, chip_id) + uri STR 255 + bg F32 0.0 + bg_stdev F32 0.0 + bg_mean_stdev F32 0.0 + sigma_ra F32 0.0 + sigma_dec F32 0.0 + zp_mean F32 0.0 + zp_stdev F32 0.0 + fwhm F32 0.0 + fwhm_range F32 0.0 + n_stars S32 0 + n_extended S32 0 + n_cr S32 0 + n_astrom S32 0 + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +camMask METADATA + label STR 64 # Primary Key +END Index: /branches/jh_branches/backtrack/dbconfig/changes.txt =================================================================== --- /branches/jh_branches/backtrack/dbconfig/changes.txt (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/changes.txt (revision 22250) @@ -0,0 +1,25 @@ +How to manually change the database between versions via mySQL commands. + +This file is generated manually, and may not be complete. + + +Version 1.1.20 --> 1.1.21: + +alter table camProcessedExp drop nastro; +alter table camProcessedExp add (fwhm double, fwhm_range double, n_stars int(11), n_extended int(11), n_cr int(11), n_astrom int(11)); +alter table chipProcessedImfile add column (bias double, bias_stdev double, fringe_0 double, fringe_1 double, fringe_2 double, sigma_ra double, sigma_dec double, ap_resid double, ap_resid_stdev double, fwhm double, fwhm_range double, n_stars int(11), n_extended int(11), n_cr int(11), n_astrom int(11)); +alter table warpSkyfile add column path_base varchar(255) NULL DEFAULT NULL after uri; +alter table diffSkyfile add column path_base varchar(255) NULL DEFAULT NULL after uri; +alter table stackSumSkyfile add column path_base varchar(255) NULL DEFAULT NULL after uri; + + +Version 1.1.22 --> 1.1.23 +# Adding support for reduction classes (which are used to specify recipes for parts of the pipeline). +# Adding 'reduction' to detRun +# Renaming 'recipe' in {chip,cam}{Pending,Processed}Exp to 'reduction' + +alter table detRun add column reduction varchar(64) NULL DEFAULT NULL after exp_type; +alter table chipPendingExp change column recipe reduction varchar(64); +alter table chipProcessedExp change column recipe reduction varchar(64); +alter table camPendingExp change column recipe reduction varchar(64); +alter table camProcessedExp change column recipe reduction varchar(64); Index: /branches/jh_branches/backtrack/dbconfig/chip.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/chip.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/chip.md (revision 22250) @@ -0,0 +1,52 @@ +# +# We are dependant on the assumption that there is only ever one exp_tag in a +# chipRun although the database does not enforce this. +# + +chipRun METADATA + chip_id S64 0 # Primary Key AUTO_INCREMENT + state STR 64 # key + workdir STR 255 + workdir_state STR 64 # key + label STR 64 # key + reduction STR 64 # Reduction class + expgroup STR 64 # key + dvodb STR 255 +END + +chipInputImfile METADATA + chip_id S64 0 # Primary Key fkey (chip_id) ref chipRun(chip_id) + exp_tag STR 64 # Primary Key fkey (exp_tag, class_id) ref rawImfile (exp_tag, class_id) + class_id STR 64 # Primary Key +END + +chipProcessedImfile METADATA + chip_id S64 0 # Primary Key fkey (chip_id, exp_tag, class_id) ref chipPendingImfile(chip_id, exp_tag, class_id) + exp_tag STR 64 # Primary Key + class_id STR 64 # Primary Key + uri STR 255 + bg F32 0.0 + bg_stdev F32 0.0 + bg_mean_stdev F32 0.0 + bias F32 0.0 + bias_stdev F32 0.0 + fringe_0 F32 0.0 + fringe_1 F32 0.0 + fringe_2 F32 0.0 + sigma_ra F32 0.0 + sigma_dec F32 0.0 + ap_resid F32 0.0 + ap_resid_stdev F32 0.0 + fwhm F32 0.0 + fwhm_range F32 0.0 + n_stars S32 0 + n_extended S32 0 + n_cr S32 0 + n_astrom S32 0 + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +chipMask METADATA + label STR 64 # Primary Key +END Index: /branches/jh_branches/backtrack/dbconfig/config.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/config.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/config.md (revision 22250) @@ -0,0 +1,5 @@ +glueforge METADATA + pkg_name STR ippdb + pkg_namespace STR ippdb + pkg_version STR 1.1.23 +END Index: /branches/jh_branches/backtrack/dbconfig/det.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/det.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/det.md (revision 22250) @@ -0,0 +1,205 @@ +detRun METADATA + det_id S64 0 # Primary Key AUTO_INCREMENT + iteration S32 0 # Key + det_type STR 64 # Key + mode STR 64 # Key + state STR 64 # Key + filelevel STR 64 + workdir STR 255 # destination for output files + camera STR 64 + telescope STR 64 + exp_type STR 64 # XXX this should be dropped + reduction STR 64 # Reduction clas + filter STR 64 + airmass_min F32 0.0 + airmass_max F32 0.0 + exp_time_min F32 0.0 + exp_time_max F32 0.0 + ccd_temp_min F32 0.0 + ccd_temp_max F32 0.0 + posang_min F64 0.0 + posang_max F64 0.0 + registered TAI 0001-01-01T00:00:00Z + time_begin TAI 0001-01-01T00:00:00Z + time_end TAI 0001-01-01T00:00:00Z + use_begin TAI 0001-01-01T00:00:00Z + use_end TAI 0001-01-01T00:00:00Z + solang_min F32 0.0 + solang_max F32 0.0 + label STR 64 # key + parent S32 0 # Key +END + +detInputExp METADATA + det_id S64 0 # Primary Key + iteration S32 0 # Primary Key + exp_tag STR 64 # Primary Key + include BOOL f +END + +detProcessedImfile METADATA + det_id S64 0 # Primary Key + exp_tag STR 64 # Primary Key + class_id STR 64 # Primary Key + uri STR 255 + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + fringe_0 F64 0.0 + fringe_1 F64 0.0 + fringe_2 F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +detProcessedExp METADATA + det_id S64 0 # Primary Key + exp_tag STR 64 # Primary Key + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + fringe_0 F64 0.0 + fringe_1 F64 0.0 + fringe_2 F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +detStackedImfile METADATA + det_id S64 0 # Primary Key + iteration S32 0 # Primary Key + class_id STR 64 # Primary Key + uri STR 255 + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + fault S16 0 # Key NOT NULL +END + +detNormalizedStatImfile METADATA + det_id S64 0 # Primary Key + iteration S32 0 # Primary Key + class_id STR 64 # Primary Key + norm F32 0.0 + fault S16 0 # Key NOT NULL +END + +detNormalizedImfile METADATA + det_id S64 0 # Primary Key + iteration S32 0 # Primary Key + class_id STR 64 # Primary Key + uri STR 255 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +detNormalizedExp METADATA + det_id S64 0 # Primary Key + iteration S32 0 # Primary Key + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +#detMasterFrame METADATA +# det_id S64 0 # Primary Key +# iteration S32 0 # Primary Key +# comment STR 255 +#END +# +## drop? +#detMasterImfile METADATA +# det_id S64 0 # Primary Key +# class_id STR 64 # Primary Key +# uri STR 255 +# recipe STR 64 +#END + +detResidImfile METADATA + det_id S64 0 # Primary Key + iteration S32 0 # Primary Key + exp_tag STR 64 # Primary Key + class_id STR 64 # Primary Key + uri STR 255 + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + bin_stdev F64 0.0 + fringe_0 F64 0.0 + fringe_1 F64 0.0 + fringe_2 F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +detResidExp METADATA + det_id S64 0 # Primary Key + iteration S32 0 # Primary Key + exp_tag STR 64 # Primary Key + recipe STR 64 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + bin_stdev F64 0.0 + fringe_0 F64 0.0 + fringe_1 F64 0.0 + fringe_2 F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + path_base STR 255 + accept BOOL f + fault S16 0 # Key NOT NULL +END + +detRunSummary METADATA + det_id S64 0 # Primary Key + iteration S32 0 # Primary Key + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + accept BOOL f + fault S16 0 # Key NOT NULL +END Index: /branches/jh_branches/backtrack/dbconfig/diff.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/diff.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/diff.md (revision 22250) @@ -0,0 +1,34 @@ +# $Id: diff.md,v 1.3 2007-06-07 01:10:31 price Exp $ + +diffRun METADATA + diff_id S64 0 # Primary Key AUTO_INCREMENT + state STR 64 # Key + workdir STR 255 + dvodb STR 255 + registered TAI NULL + skycell_id STR 64 # Key + tess_id STR 64 # Key +END + +# +# Diff Sky Cells Mode +# + +# only ever 2 per run +diffInputSkyfile METADATA + diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) + warp_id S64 0 # Primary Key fkey(warp_id, skycell_id, tess_id) ref warpSkyfile(warp_id, skycell_id, tess_id) + skycell_id STR 64 # Primary Key + tess_id STR 64 # Primary Key +# either a input or a template + kind STR 64 # Key + template BOOL f +END + +diffSkyfile METADATA + diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) + uri STR 255 + path_base STR 255 + bg F64 0.0 + bg_stdev F64 0.0 +END Index: /branches/jh_branches/backtrack/dbconfig/dimm.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/dimm.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/dimm.md (revision 22250) @@ -0,0 +1,21 @@ +# Table 13: DIMM Measurements Table +# Column Name Datatype Description +# Time date/time The time the DIMM observation was taken. +# sigmax float Raw dispersion in x. +# sigmay float Raw dispersion in y. +# FWHM float Dervied seeing full width at half maximum. +# RA float The coordinates of the measured star. +# DEC float The coordinates of the measured star. +# Exposure time float The exposure time of the DIMM observation. +# Telescope ID string source of the DIMM data + +dimm METADATA +# time DATETIME 2006-01-11T00:00:00 + sigmax F32 0.0 + sigmay F32 0.0 + fwhm F32 0.0 + ra F64 0.0 + decl F64 0.0 + expttime F32 0.0 + telescope_id STR 255 +END Index: /branches/jh_branches/backtrack/dbconfig/dome.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/dome.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/dome.md (revision 22250) @@ -0,0 +1,16 @@ +# Table 15: Dome Status Table +# Column Name Datatype Description +# Time date/time The time for which the dome status is valid. +# Azimuth float The azimuth of the dome. +# Open status boolean Whether the dome is open or not. +# Lights status boolean Whether lights are on in the dome or not. +# Track status boolean Whether dome is tracking telescope or not. + +dome METADATA +# time DATETIME 2006-01-11T00:00:00 + az F32 0.0 + open BOOL t + light BOOL t + # XXX is it possible for the dome slit to not track the telescope? ;) + track BOOL t +END Index: /branches/jh_branches/backtrack/dbconfig/guide.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/guide.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/guide.md (revision 22250) @@ -0,0 +1,5 @@ +guidePendingExp METADATA + guide_id S64 0 # Primary Key AUTO_INCREMENT + exp_tag STR 64 # Key + recipe STR 64 +END Index: /branches/jh_branches/backtrack/dbconfig/ipp.m4 =================================================================== --- /branches/jh_branches/backtrack/dbconfig/ipp.m4 (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/ipp.m4 (revision 22250) @@ -0,0 +1,18 @@ +include(config.md) +dnl include(weather.md) +dnl include(skyp_transparency.md) +dnl include(skyp_absorption.md) +dnl include(skyp_emission.md) +dnl include(dimm.md) +dnl include(skyp_ir.md) +dnl include(dome.md) +dnl include(telescope.md) +include(tasks.md) +include(guide.md) +include(chip.md) +include(cam.md) +include(warp.md) +include(diff.md) +include(stack.md) +include(det.md) +dnl include(skycell.md) Index: /branches/jh_branches/backtrack/dbconfig/magic.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/magic.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/magic.md (revision 22250) @@ -0,0 +1,85 @@ +# $Id: magic.md,v 1.1 2007-02-15 02:12:42 jhoblitt Exp $ + +# +# Feed imfiles to Magic Mode +# + +# also uses p4InputExp + +# output from magic +p4MagicMaskImfile METADATA + p4_id S32 0 # Primary Key + exp_tag STR 64 # Primary Key + p3_version S32 0 # Primary Key +# class is not yet consistently carried through pXtools +# class STR 64 # Primary Key + class_id STR 64 # Primary Key + uri STR 255 +END + +# output from magic +# p4MagicMaskImfile METADATA +# p4_id S32 0 # Primary Key +# exp_tag STR 64 # Primary Key +# p3_version S32 0 # Primary Key +# # class is not yet consistently carried through pXtools +# # class STR 64 # Primary Key +# class_id STR 64 # Primary Key +# uri STR 255 +# END +# +# # the list of imfiles in our skycell keyed against phase 3 +# p4InputImfile METADATA +# p4_id S32 0 # Primary Key +# skycell_id STR 64 # Primary Key +# tess_id STR 64 # Primary Key +# exp_tag STR 64 # Primary Key +# p3_version S32 0 # Primary Key +# # class is not yet consistently carried through pXtools +# # class STR 64 # Primary Key +# class_id STR 64 # Primary Key +# # either a input or a template +# kind STR 64 # Key +# END +# +# +# # one diff image per input to the current stack +# # XXX how will diff against the static sky be handled? +# p4DiffImfile METADATA +# p4_id S32 0 # Primary Key +# skycell_id STR 64 # Primary Key +# tess_id STR 64 # Primary Key +# exp_tag STR 64 # Primary Key +# p3_version S32 0 # Primary Key +# # class is not yet consistently carried through pXtools +# # class STR 64 # Primary Key +# class_id STR 64 # Primary Key +# uri STR 255 +# bg F64 0.0 +# bg_mean_stdev F64 0.0 +# END +# +# p4StackedImfile METADATA +# p4_id S32 0 # Primary Key +# skycell_id STR 64 # Primary Key +# tess_id STR 64 # Primary Key +# exp_tag STR 64 # Primary Key +# p3_version S32 0 # Primary Key +# # class is not yet consistently carried through pXtools +# # class STR 64 # Primary Key +# class_id STR 64 # Primary Key +# uri STR 255 +# bg F64 0.0 +# bg_mean_stdev F64 0.0 +# END +# +# # output from magic +# p4MagicMaskImfile METADATA +# p4_id S32 0 # Primary Key +# exp_tag STR 64 # Primary Key +# p3_version S32 0 # Primary Key +# # class is not yet consistently carried through pXtools +# # class STR 64 # Primary Key +# class_id STR 64 # Primary Key +# uri STR 255 +# END Index: /branches/jh_branches/backtrack/dbconfig/notes.txt =================================================================== --- /branches/jh_branches/backtrack/dbconfig/notes.txt (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/notes.txt (revision 22250) @@ -0,0 +1,41 @@ + +2007.06.08 : EAM + + * when changing the database schema: + * increment the pkg_version number on dbconfig/config.md + * increment the ippdb version number in ippTools/configure.ac (to match) + * increment the ippTools version number in ippTools/configure.ac + * build ippbd ('make src' in dbconfig) + * check in dbconfig, ippdb, and ippTools + +2007.06.04 : EAM + +I am adding additional fields which we will need for selecting the +input detrend images: + +rawExp : + user stat 1 + user stat 2 + user stat 3 + user stat 4 + user stat 5 + +detProcessedImfile, detProcessedExp: + fringe amplitude + user stat 1 + user stat 2 + user stat 3 + user stat 4 + user stat 5 + +detResidImfile, detResidExp: + binned stdev + fringe_0 + fringe_1 + fringe_2 + user stat 1 + user stat 2 + user stat 3 + user stat 4 + user stat 5 + Index: /branches/jh_branches/backtrack/dbconfig/skycell.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/skycell.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/skycell.md (revision 22250) @@ -0,0 +1,25 @@ +# $Id: skycell.md,v 1.3 2007-02-13 22:33:30 jhoblitt Exp $ + +skyCell METADATA + skycell_id STR 64 # Primary Key + tess_id STR 64 # Primary Key + ra1 F64 0.0 + decl1 F64 0.0 + ra2 F64 0.0 + decl2 F64 0.0 + ra3 F64 0.0 + decl3 F64 0.0 + ra4 F64 0.0 + decl4 F64 0.0 +END + +skyCellMap METADATA + skycell_id STR 64 # Primary Key + tess_id STR 64 # Primary Key + exp_tag STR 64 # Primary Key + p3_version S32 0 # Primary Key +# class is not yet consistently carried through pXtools +# class STR 64 # Primary Key + class_id STR 64 # Primary Key +END + Index: /branches/jh_branches/backtrack/dbconfig/skyp_absorption.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/skyp_absorption.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/skyp_absorption.md (revision 22250) @@ -0,0 +1,25 @@ +# Table 11: Skyprobe Line Absorption Table (sample entries) +# Column Name Datatype Description +# Time date/time The time the LRProbe observation was taken. +# Disperser ID string ID of the dispersing element +# Atm Component 1 float The strength of the 1st atmospheric component. +# Atm Component 2 float The strength of the 2nd atmospheric component. +# Atm Component 3 float The strength of the 3rd atmospheric component. +# Disperser ID string ID of the dispersing element +# Number of stars int Number of stars used to measure the absorptions. +# Astrometry coords The astrometry used on the LRProbe image. +# Exposure time float The exposure time of the LRProbe image. +# Sky brightness float The measured sky (surface) brightness, in physical units. + +skyp_absorption METADATA +# time DATETIME 2006-01-10T00:00:00 + disperser_id STR 255 + atmcomp1 F32 0.0 + atmcomp2 F32 0.0 + atmcomp3 F32 0.0 + nstars S32 0 + ra F64 0.0 + decl F64 0.0 + exptime F32 0.0 + sky_bright F64 0.0 +END Index: /branches/jh_branches/backtrack/dbconfig/skyp_emission.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/skyp_emission.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/skyp_emission.md (revision 22250) @@ -0,0 +1,20 @@ +# Table 12: Skyprobe Line Emission Table (sample entries) +# Column Name Datatype Description +# Time date/time The time the LRProbe observation was taken. +# Disperser ID string ID of the dispersing element +# Atm Component 1 float The strength of the 1st atmospheric component. +# Atm Component 2 float The strength of the 2nd atmospheric component. +# Atm Component 3 float The strength of the 3rd atmospheric component. +# Continuum float The strength of the continuum emission. +# Disperser ID string ID of the dispersing element +# Exposure time float The exposure time of the LRProbe image. + +skyp_emission METADATA +# time DATETIME 2006-01-11T00:00:00 + disperser_id STR 255 + atmcomp1 F32 0.0 + atmcomp2 F32 0.0 + atmcomp3 F32 0.0 + continuum F32 0.0 + exptime F32 0.0 +END Index: /branches/jh_branches/backtrack/dbconfig/skyp_ir.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/skyp_ir.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/skyp_ir.md (revision 22250) @@ -0,0 +1,18 @@ +# Table 14: Near IR Wide-field Camera Results Table +# Column Name Datatype Description +# Time date/time The time the NIR observation was taken. +# Sky brightness float The sky (surface) brightness in the NIR observation. +# Sky variance float The variance in the sky (surface) brightness. +# Astrometry coords The astrometry used on the NIR image. +# FOV X float field width +# FOV Y float field height + +skyp_ir METADATA +# time DATETIME 2006-01-11T00:00:00 + sky_bright F64 0.0 + sky_var F64 0.0 + ra F64 0.0 + decl F64 0.0 + fov_x F32 0.0 + fov_y F32 0.0 +END Index: /branches/jh_branches/backtrack/dbconfig/skyp_transparency.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/skyp_transparency.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/skyp_transparency.md (revision 22250) @@ -0,0 +1,21 @@ +# Table 10: SkyProbe Transparency Table (sample entries) +# Column Name Datatype Description +# Time date/time The time the SkyProbe image was taken. +# Filter string Filter used for SkyProbe image. +# Transparency float The derived transparency. +# Number of stars int The number of stars used to measure the transparency. +# Astrometry coords The astrometry used on the SkyProbe image. +# Exposure time float The exposure time of the SkyProbe image. +# Sky brightness float The measured sky (surface) brightness, counts / second + +skyp_transparency METADATA +# time DATETIME 2006-01-10T00:00:00 # Primary Key + filter STR 255 + trans F64 0.0 + nstars S32 0 +# astrom + ra F64 0.0 + decl F64 0.0 + exptime F32 0.0 + sky_bright F64 0.0 +END Index: /branches/jh_branches/backtrack/dbconfig/stack.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/stack.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/stack.md (revision 22250) @@ -0,0 +1,24 @@ +# $Id: stack.md,v 1.3 2007-06-07 01:10:31 price Exp $ + +stackRun METADATA + stack_id S64 0 # Primary Key AUTO_INCREMENT + state STR 64 # Key + workdir STR 255 + dvodb STR 255 + registered TAI NULL + skycell_id STR 64 # Key + tess_id STR 64 # Key +END + +stackInputSkyfile METADATA + stack_id S64 0 # Primary Key fkey(stack_id) ref stackRun(stack_id) + warp_id S64 0 # Primary Key fkey(warp_id) ref warpSkyfile(warp_id) +END + +stackSumSkyfile METADATA + stack_id S64 0 # Primary Key fkey(stack_id) ref stackRun(stack_id) + uri STR 255 + path_base STR 255 + bg F64 0.0 + bg_stdev F64 0.0 +END Index: /branches/jh_branches/backtrack/dbconfig/tasks.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/tasks.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/tasks.md (revision 22250) @@ -0,0 +1,171 @@ +# $Id: tasks.md,v 1.134.4.2 2007-06-25 20:27:07 jhoblitt Exp $ + +# this table records all exposure ID ever seen from the summit +# exp_id == fileset + +# note that dec is a MySQL reserved word +# note that use is a MySQL reserved word +# note that exp is a MySQL reserved word + +# for use with this stored procedure to generate exp_tags under Mysql 5+ +# +#CREATE FUNCTION genTag (exp_id varchar(64)) RETURNS VARCHAR(64) +#BEGIN +# UPDATE expTagCounter SET counter = LAST_INSERT_ID(counter + 1); +# RETURN CONCAT(exp_id, '.', LAST_INSERT_ID()); +#END +# +expTagCounter METADATA + counter U64 0 +END + +# list of source exposures -- updated as exposures are seen +# summitExp.imfiles is updated as filesets are queried default value should be +# -1 as NULL or 0 might be a valid value (empty fileset) +summitExp METADATA + exp_id STR 64 # Primary Key + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + dateobs UTC NULL + exp_type STR 64 + uri STR 255 + imfiles S32 0 +END + +# class == type of file +# class_id == type set id +# list of source images -- updated as exposures/filesets are queried +summitImfile METADATA + exp_id STR 64 # Primary Key + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + file_id STR 64 # Key + bytes S32 0 + md5sum STR 32 + class STR 64 # Primary Key + class_id STR 64 # Primary Key + uri STR 255 +END + +# list of exposures that have had their imfiles/files registered (but not +# downloaded) +pzPendingExp METADATA + exp_id STR 64 # Primary Key + camera STR 64 # Primary Key + telescope STR 64 # Primary Key +END + +pzPendingImfile METADATA + exp_id STR 64 # Primary Key + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + class STR 64 # Primary Key + class_id STR 64 # Primary Key + exp_tag STR 64 # Unique Key +END + +# list of exposures that have had all of their imfiles downloaded +pzDoneExp METADATA + exp_id STR 64 # Primary Key + camera STR 64 # Primary Key + telescope STR 64 # Primary Key +END + +pzDoneImfile METADATA + exp_id STR 64 # Primary Key + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + class STR 64 # Primary Key + class_id STR 64 # Primary Key + exp_tag STR 64 # Unique Key + uri STR 255 +END + +newExp METADATA + exp_tag STR 64 # Primary Key + exp_id STR 64 # Key + state STR 64 # Key + camera STR 64 # Key + telescope STR 64 # Key + imfiles S32 0 + workdir STR 255 # destination for output files + workdir_state STR 64 # key +END + +# class needs to be carried here so it can go into rawImfile and be normalized +# from there +newImfile METADATA + exp_tag STR 64 # Primary Key fkey(exp_tag) ref newExp(exp_tag) + class_id STR 64 # Primary Key + uri STR 255 +END + +# paired with rawImfile +rawExp METADATA + exp_tag STR 64 # Primary Key fkey(exp_tag) ref newExp(exp_tag) + exp_id STR 64 # Key + camera STR 64 + telescope STR 64 + dateobs UTC 0001-01-01T00:00:00Z + exp_type STR 64 + imfiles S32 0 + filelevel STR 64 + workdir STR 255 # destination for output files + filter STR 64 + airmass F32 0.0 + ra F64 0.0 + decl F64 0.0 + exp_time F32 0.0 + sat_pixel_frac F32 0.0 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + alt F64 0.0 + az F64 0.0 + ccd_temp F32 0.0 + posang F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + object STR 64 + solang F32 0.0 + fault S16 0 # Key NOT NULL +END + +rawImfile METADATA + exp_tag STR 64 # Primary Key fkey(exp_tag, class_id) ref newImfile(exp_tag, class_id) + class_id STR 64 # Primary Key + uri STR 255 + exp_type STR 64 +# This field is used to set the per exp filelevel. Thus the values for all of +# the imfiles in an exposure need to be sanity checked to make sure that this +# value is in argeement. + filelevel STR 64 + filter STR 64 + airmass F32 0.0 + ra F64 0.0 + decl F64 0.0 + exp_time F32 0.0 + sat_pixel_frac F32 0.0 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + alt F64 0.0 + az F64 0.0 + ccd_temp F32 0.0 + posang F64 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + object STR 64 + dateobs UTC 0001-01-01T00:00:00Z + fault S16 0 # Key NOT NULL +END + +# +# XXX - temporarily (???) stop carrying around class in imfiles after rawImfile +# Index: /branches/jh_branches/backtrack/dbconfig/telescope.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/telescope.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/telescope.md (revision 22250) @@ -0,0 +1,18 @@ +# Table 16: Telescope Status +# Column Name Datatype Description +# Time date/time The time for which the telescope status is valid. +# Guide status enum The status of the guiding. +# Altitude float The telescope altitude. +# Azimuth float The telescope azimuth. +# RA float The telescope Right Ascension (ICRS ~ J2000). +# Dec float The telescope Declination (ICRS ~ J2000). + +telescope METADATA +# time DATETIME 2006-01-11T00:00:00 # Primary Key +# XXX there is currently no way to declare an enum - use str or int instead? + guide STR 255 + alt F32 0.0 + az F32 0.0 + ra F64 0.0 + decl F64 0.0 +END Index: /branches/jh_branches/backtrack/dbconfig/warp.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/warp.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/warp.md (revision 22250) @@ -0,0 +1,53 @@ +# $Id: warp.md,v 1.3 2007-06-07 01:10:31 price Exp $ + +# +# We have at least 3 different run types +# a) single epoch differing +# b) magic run operates on "complete" exposures +# c) multiple epoch differing/stacking operations +# + +# define a new warprun for a single skycell +warpRun METADATA + warp_id S64 0 # Primary Key AUTO_INCREMENT + mode STR 64 # Key + state STR 64 # Key + workdir STR 255 + dvodb STR 255 + registered TAI NULL +END + +# +# Warp to Sky Cell Mode +# + +# the list of imfiles in our skycell keyed against phase 3 +# only allow one cam_version of an exp_tag per warpRun +warpInputExp METADATA + warp_id S64 0 # Primary Key fkey(warp_id) ref warpRun(warp_id) + cam_id S64 0 # Primary Key fkey(cam_id) ref camProcessedExp(cam_id) +# if magic is T then look for the exp_tag in the magic output tables + magiced BOOL f # Key +END + +warpSkyCellMap METADATA + warp_id S64 0 # Primary Key fkey(warp_id, cam_id) ref warpInputExp(warp_id, cam_id) + skycell_id STR 64 # Primary Key + tess_id STR 64 # Primary Key + cam_id S64 0 # Primary Key +# class is not yet consistently carried through pXtools +# class STR 64 # Primary Key + class_id STR 64 # Primary Key + fault S16 0 # Key +END + +warpSkyfile METADATA + warp_id S64 0 # Primary Key fkey(warp_id, skycell_id, tess_id) ref warpSkyCellMap(warp_id, skycell_id, tess_id) + skycell_id STR 64 # Primary Key + tess_id STR 64 # Primary Key + uri STR 255 + path_base STR 255 + bg F64 0.0 + bg_stdev F64 0.0 +END + Index: /branches/jh_branches/backtrack/dbconfig/weather.md =================================================================== --- /branches/jh_branches/backtrack/dbconfig/weather.md (revision 22250) +++ /branches/jh_branches/backtrack/dbconfig/weather.md (revision 22250) @@ -0,0 +1,19 @@ +# Table 9: Weather Table: some sample weather points +# Column Name Datatype Description +# Time date/time The time the weather information was measured. +# Temperature 01 float The external temperature +# Temperature 02 float The temperature at top of the dome +# Temperature 03 float The temperature on the primary mirror +# Humidity float The relative humidity. +# Pressure float The (external) atmospheric pressure. + +weather METADATA +# time DATETIME 2006-01-10T00:00:00 # Primary Key + temp01 F32 0.0 + humi01 F32 0.0 + temp02 F32 0.0 + humi02 F32 0.0 + temp03 F32 0.0 + humi03 F32 0.0 + pressure F32 0.0 +END Index: /branches/jh_branches/backtrack/ippTools/.cvsignore =================================================================== --- /branches/jh_branches/backtrack/ippTools/.cvsignore (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/.cvsignore (revision 22250) @@ -0,0 +1,16 @@ +Makefile +Makefile.in +aclocal.m4 +autom4te.cache +compile +config.guess +config.log +config.status +config.sub +configure +depcomp +install-sh +libtool +ltmain.sh +missing +pxtools.pc Index: /branches/jh_branches/backtrack/ippTools/COPYING =================================================================== --- /branches/jh_branches/backtrack/ippTools/COPYING (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/COPYING (revision 22250) @@ -0,0 +1,345 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. Index: /branches/jh_branches/backtrack/ippTools/Makefile.am =================================================================== --- /branches/jh_branches/backtrack/ippTools/Makefile.am (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/Makefile.am (revision 22250) @@ -0,0 +1,12 @@ +SUBDIRS = src share + +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA= pxtools.pc + +EXTRA_DIST = \ + pxtools.pc.in \ + autogen.sh + +CLEANFILES = *~ core core.* + +ACLOCAL_AMFLAGS = -I m4 Index: /branches/jh_branches/backtrack/ippTools/TODO =================================================================== --- /branches/jh_branches/backtrack/ippTools/TODO (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/TODO (revision 22250) @@ -0,0 +1,14 @@ +- change dettol -pretend -define... to return a metadata structure that shows what the created detRun would look like +- rename dettool -residdetrun -> -todetrunsummary +- add foreign key constraints to the database +- add the ability to remove dettool detResidExps +- change p4tool to accept input from/with magic masks +- change p5tool to accept input from warp * stack +- difftool/stacktool error handling +- move all large SQL statements out into separate files +- add the option of not automatically flowing through from reg -> chip -> cam + tools +- add workdir inheritance rooted in rawExp, eg. + select chipProcessedExp.workdir, rawExp.workdir, IFNULL(chipProcessedExp.workdir, rawExp.workdir) from chipProcessedExp join rawExp using(exp_tag); +- dettool/detRun support for filelevel +- combind dettool -definebytag & -definebyquery Index: /branches/jh_branches/backtrack/ippTools/autogen.sh =================================================================== --- /branches/jh_branches/backtrack/ippTools/autogen.sh (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/autogen.sh (revision 22250) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=pxtools +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +#($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $LIBTOOlIZE installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" +# DIE=1 +#} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +#($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $AUTOHEADER installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" +# DIE=1 +#} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +$LIBTOOLIZE --copy --force || echo "$LIBTOOlIZE failed" +$ACLOCAL -I m4 || echo "$ACLOCAL failed" +$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: /branches/jh_branches/backtrack/ippTools/configure.ac =================================================================== --- /branches/jh_branches/backtrack/ippTools/configure.ac (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/configure.ac (revision 22250) @@ -0,0 +1,64 @@ +AC_PREREQ(2.59) + +AC_INIT([ipptools], [1.1.23], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([autogen.sh]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_CONFIG_HEADER([src/config.h]) +AM_MAINTAINER_MODE + +IPP_STDCFLAGS + +AC_LANG(C) +AC_GNU_SOURCE +AC_PROG_CC +AC_PROG_INSTALL +AC_PROG_LIBTOOL + +PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0]) +PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) +PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.23]) + +PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}" +PXTOOLS_LIBS="${PSLIB_LIBS=} ${PSMODULES_LIBS=} ${IPPDB_LIBS=}" +AC_SUBST(PXTOOLS_CFLAGS,[$PXTOOLS_CFLAGS]) +AC_SUBST(PXTOOLS_LIBS,[$PXTOOLS_LIBS]) + +dnl check for PSDB support +TMP_CFLAGS=${CFLAGS} +CFLAGS="${CFLAGS=} ${PSLIB_CFLAGS}" + +AC_COMPILE_IFELSE([AC_LANG_PROGRAM( + [[#if !defined HAVE_PSDB + error: PSDB support is not avaiable + #endif + ]])], + [], + [AC_MSG_ERROR([psLib was built without PSDB support, bailing out.])] +) + +CFLAGS=${TMP_CFLAGS} + +dnl -- check for psParseErrorCodes -------------------------------------- + +AC_PATH_PROG([ERRORCODES], [psParseErrorCodes], [missing]) +if test "$ERRORCODES" = "missing" ; then + AC_MSG_ERROR([psParseErrorCodes is required]) +fi + +dnl check for perl +AC_PATH_PROG([PERL], [perl], [missing]) +if test "$PERL" = "missing" ; then + AC_MSG_ERROR([perl is required]) +fi + +IPP_STDOPTS +CFLAGS="${CFLAGS=} -Wall -Werror -std=c99" + +AC_CONFIG_FILES([ + Makefile + src/Makefile + share/Makefile + pxtools.pc +]) +AC_OUTPUT Index: /branches/jh_branches/backtrack/ippTools/doc/dettool.txt =================================================================== --- /branches/jh_branches/backtrack/ippTools/doc/dettool.txt (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/doc/dettool.txt (revision 22250) @@ -0,0 +1,181 @@ +# returns a list of unassociated rawDetrendExp (rawDetrendExps that don't +# correspond to a exp_id in detInputExp) +dettool -pending -exp_type -inst .. + +# define a new detRun semi-automatically by selecting the component exps with +# the specified search criteria +dettool -definebyquery -det_type ... + +# create a detRun with the specified input exp_ids +dettool -definebyexp -det_type -exp_id 1 -exp_id 2 ... + +-- +# returns a list of rawImfiles that are associated with an detInputExp that do +# not appear in detProcessedImfiles +dettool -raw + +# add an imfile to detProcessedImfiles +dettool -addprocessed + +-- +# the list of class_ids that are ready to be stacked +# returns a list of det_id/iter/class_id/det_typs where class_id is unique per +# det_id (not a list of all exp_ids/class_ids) that do not have an entry in +# detStackedImfile +# results are masked by entries in detResidExpAnalysis +# may need to check detResidExp's even on iter 0 - ??? +dettool -tostack + +# list all of imfiles from different exps with the same class_id that have +# been processed +dettool -processed -wholestack -det_id ... + +# adds a stacked class_id and and optionally marks it for normalization +detool -addstacked [-pleasenormalize] -det_id -iter -class_id ... + +-- +# returns a list of det_ids/iters where the entire set of class_ids has been +# processed +dettool -tonormalize + +# returns the complete list of processed imfiles for the specified det_id +dettool -processed -unmask -det_id -iter + +# add per stacked imfile normalization statistics +# O: this could accept a MDC doc as input from stdin +dettool -addnormstat -det_id -iter -class_idi -norm F32 + +-- +# returns a list of det_ids/iter/class_id/uri from detNormStats. Stops +# returning entries after the class_is is inserted into detNormalizedImfile +dettool -normstat -det_id + +# inserts an entry into detNormalizedImfile +dettool -addnormalized -det_id -ter -class_id -uri + +-- +# returns a list of processed imfiles that have also been normalized with +# processed imfiles being masked out per detResidImfileAnalysis +# also returns stackedimfiles that have -pleasenormalize set to false +dettool -toresid -det_id -iter + +# inserts a per det_id/iter/exp_id/class_id residual +dettool -addresidimfile -det_id -exp_id -class_id -iter -stat F32 -stat_stdev F32 -uri -b1 -b2 + +# returns a list of det_id/iter/exp_ids but only for exp_ids that have resids +# for all of their imfiles +dettool -toresidexp + +# returns a list of residual imfile data +dettool -residimfile -det_id -iter -exp_id + +# inserts into detResidExpAnalysis +# sets the accept accept bool unless -reject is specified +dettool -addresidexp -det_id -iter -exp_id -jpeg1 -jpeg2 -stat F32 -stat_stdev +F32 [-reject] + +-- +# lists det_id/iter for detruns that have completed all residexps for their +# current iteration +dettool -residdetrun + +# lists all residexps for the given det_id/iter +dettool -residexp -det_id -iter + +# updates the given residexp to be accept/reject +dettool -updateresid -det_id -iter [-reject] + +# changes the state of the detrun. -rerun cause a new iteration to be started +# with the accepted with just the exp_ids +dettool -updatedetrun -det_id -iter -stat F32 -stat_stdev [-rerun|-ok|-stop] + +# manual starts a new detrun iteration starting from the specified -iter +dettool -rerun -det_id -iter -accept expid -accept expid -reject expid -reject expid +dettool -rerun -det_id -iter + +# detInputExp needs to track... +det_id iter exp_id use bool accept bool + + + +dettool + + -define | creates a a new detrend Run + input: + exp IDs? + type of det run? + creates a new detRun ID automatically + + output: the new det_id + + -raw | lists raw imfiles needing to be processed + input: + det ID + exp ID + class ID + iteration number? + output: list of unprocessed imfiles + + -addprocessed? | marks a raw imfile as processed + input: + det ID + exp ID + class ID + iteration number? + uri + stat + recipe + output: outout on error only + -processed | lists processed imfiles + input: + det ID + exp ID + class ID + iteration number? + output: list of processed imfiles + -addstacked | adds a stacked imfiles + input: + det ID + class ID + iteration number? + URI + recipe + stats + output: output on error only + -stacked | lists stacked imfiles + input: + det ID + class ID + iteration number? + output: list of stacked imfiles + -stacked | lists stacked imfiles + input: + det ID + class ID + iteration number? + output: list of stacked imfiles + -stackedframe | lists stacked imfiles for COMPLETE frames + input: + det ID // required for simplicity + iteration number? + output: list of stacked imfiles + -addmaster | adds a master imfiles + input: + det ID + class ID + iteration number? + URI + recipe + stats + output: output on error only + -master | lists master imfiles + input: + det ID // required for simplicity + class ID + iteration number? + output: a list of master imfiles + -masterframe | lists master imfiles for COMPLETE frames + input: + det ID // required for simplicity + iteration number? + output: a list of master imfiles Index: /branches/jh_branches/backtrack/ippTools/doc/dsfilesetls =================================================================== --- /branches/jh_branches/backtrack/ippTools/doc/dsfilesetls (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/doc/dsfilesetls (revision 22250) @@ -0,0 +1,6 @@ +#!/usr/bin/env perl + +print "# uri fileid bytes md5sum type \n"; +print "http://example.org/1 foo1.0 123 asadfasdfasdfasfdasdf CELL\n"; +print "http://example.org/2 foo1.1 123 asadfasdfasdfasdfasdf CELL\n"; +print "http://example.org/3 foo1.2 123 asadfasdfasdfasdfasdf CELL\n"; Index: /branches/jh_branches/backtrack/ippTools/doc/dsproductls =================================================================== --- /branches/jh_branches/backtrack/ippTools/doc/dsproductls (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/doc/dsproductls (revision 22250) @@ -0,0 +1,8 @@ +#!/usr/bin/env perl + +print "# uri fileset datetime type\n"; +print "http://example.org/1 foo1 2006-04-17T00:00:12 OBJECT\n"; +print "http://example.org/2 foo2 2006-04-17T00:00:12 OBJECT\n"; +print "http://example.org/3 foo3 2006-04-17T00:00:12 OBJECT\n"; + + Index: /branches/jh_branches/backtrack/ippTools/doc/install.pod =================================================================== --- /branches/jh_branches/backtrack/ippTools/doc/install.pod (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/doc/install.pod (revision 22250) @@ -0,0 +1,251 @@ +=pod + +=head1 SYNOPSIS + +C is a collection of utilties for tracking image processing +tasks. Very little decision making logic is built into these programs. The +intent is for higher level programs and scripts to make the real I +and then use C to encode those choices in a persistent manner. + +=head1 INSTALL + +=head2 Prerequisites + +C depends on C (built with C support), C, and +C and requires a C capable compiler. + +XXX document required library versions + +=head2 Build Procedure + +C is built with the so-called suite of autotools. The build +procedure is pretty typical for software built with these tools. E.g. + + ./configure + make + make install + +If you checked out the sources directly from CVS you need to run the +C script. C will run C for you so that +changes the build procedure to just: + + sh autogen.sh + make + make install + +=head2 jhbuild Build Procedure + +If you are using one of the C modulesets jhbuild can build and install C for you. E.g. + + jhbuild build pxtools + +=head2 Database Setup + +All of the C' persistent storage is ultimately done through C's +C facilties. In turn, C uses C to impliment storage (and +relational querying). C requires that each namespace (aka database) is +configured with basic access controls. These include usernames, passwords, +access filtering, and privilege restrictions. Below is a simple (but somewhat +dangerous) example of configuring a database named C. + + mysql -u root mysql -p + (enter your MySQL root password) + + DROP DATABASE IF EXISTS ipp; + CREATE DATABASE ipp; + GRANT ALL PRIVILEGES ON *.* TO 'ippuser'@'localhost' IDENTIFIED BY 'ipppass'; + FLUSH PRIVILEGES; + quit + +After setting up the database you should verify that you can connect to with the username and password you configured + + mysql -u ipp ipp -pipp + +Next, C needs to be told where to to find MySQL, the name of the +database you created, the username, and the password you created. C +uses the typical C configuration system. The simplest method is to +add the follow lines to your $HOME/.ipprc file. + + ### Database configuration + DBSERVER STR localhost # Database host name (for psDBInit) + DBNAME STR ipp + DBUSER STR ipp # Database user name (for psDBInit) + DBPASSWORD STR ipp # Database password (for psDBInit) + +The C utilities require the a schema is pre-loaded into the database. The C tool can do this for you. E.g. + + pxadmin -create + +You should always verify that the scheme was actually loaded. E.g. + + mysql -u ipp ipp -pipp + Welcome to the MySQL monitor. Commands end with ; or \g. + Your MySQL connection id is 6 to server version: 4.1.20-log + + Type 'help;' or '\h' for help. Type '\c' to clear the buffer. + + mysql> show tables; + +------------------------+ + | Tables_in_ipp | + +------------------------+ + | detInputExp | + | detMasterFrame | + | detMasterImfile | + | detNormalizedImfile | + | detProcessedImfile | + | detResidExpAnalysis | + | detResidImfileAnalysis | + | detRun | + | detStackedImfile | + | newExp | + | newImfile | + | p1PendingExp | + | p2DoneExp | + | p2DoneImfile | + | p2PendingExp | + | p2PendingImfile | + | p3PendingExp | + | pzPendingExp | + | pzPendingImfile | + | rawDetrendExp | + | rawImfile | + | rawScienceExp | + | summitExp | + +------------------------+ + 23 rows in set (0.00 sec) + +=head1 USE + +The workflow is as follows: + +XXX + +=head2 Phase 0 tools + +The primary C utility for handling Phase 0 is C. This tool +has two modes C<-pending> and C<-update>. The pending mode lists all +Cs that are part of an exposure that I yet been classified +as either science or detrend data. + +For example: + + p0search -pending + +Would output something like: + + newImfile MULTI # + + newImfile METADATA + exp_id STR 10 + class STR OTA + class_id STR 0 + uri STR file://0 + END + + newImfile METADATA + exp_id STR 10 + class STR OTA + class_id STR 1 + uri STR file://1 + END + + newImfile METADATA + exp_id STR 10 + class STR OTA + class_id STR 2 + uri STR file://2 + END + + newImfile METADATA + exp_id STR 10 + class STR OTA + class_id STR 3 + uri STR file://3 + END + +To declare C as science data, you would invoke C like this: + + p0search -update -exp_id 10 + +Note that by default C<-update> flags the exposure as science data. To declare it to be detrend data the command would be: + + p0search -update -exp_id 10 -detrend + +Also keep in mind that this is a one time declaration. Once it has been +declared as either science or detrend data this decision can not be undone +(without manually editing the database) + +=head2 Phase 1 tools + +These tools are not yet in use. + +=head2 Phase 2 tools + +C handles all of state for Phase 2. It has four basic modes: + +=over 4 + +=item * C<-quick> + +=item * C<-define> + +=item * C<-pending> + + p0search -pending + +=item * C<-done> + +=head2 Administrative tools + +=over 4 + +=item * C + +This tool can be used to load/unload the database schema. It has three modes of operation: + +=over 4 + +=item * C<-create> + +Loads the database schema. + +=item * C<-delete> + +Deletes the database schema. + +=item * C<-recreate> + +Deletes the database schema then reloads it. + +=back + +=item * C + +I data directly into the database. It has one mode parameter for each database table. Note: support for only a few tables has been impliemented. + +=back + +=head2 Bypassing 'Summit Copy' + +The C suite assumes that exposures and image data are initially +registered with the C set of I tools. Sometimes this is +inconvenient as you want to work with local data and bypass this set. This is +what the C utility is for. C directly inserts data into +the database, side stepping any ordering rules or sanity checking. + +The end result of the C utilities is that new exposures are registered in the C table and raw image data is in the C table. You can insert this data yourself, preparing the system to run I. + +Here is an example of inserting an exposure that has 4 component image files. + + #!//bin/sh + + inject="./pxinject" + + $inject -newExp -exp_id 10 -inst gpc -telescope ps1 -exp_type object -imfiles 4 + + `$inject -newImfile -exp_id 10 -class OTA -class_id 0 -uri file://0` + `$inject -newImfile -exp_id 10 -class OTA -class_id 1 -uri file://1` + `$inject -newImfile -exp_id 10 -class OTA -class_id 2 -uri file://2` + `$inject -newImfile -exp_id 10 -class OTA -class_id 3 -uri file://3` + +=cut Index: /branches/jh_branches/backtrack/ippTools/doc/p0tools.txt =================================================================== --- /branches/jh_branches/backtrack/ippTools/doc/p0tools.txt (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/doc/p0tools.txt (revision 22250) @@ -0,0 +1,40 @@ + +Phase 0 pipeline tools: + +p0search -pending : + * examine the new.imfiles,new.exposures tables and select exposures ready for analysis + * output is: (expID) (camera) + +p0search -update (expID) (camera): + * select a the corresponding images from the new.imfiles/new.exposures table + * extract the specified header information + * search the summit metadata db tables + * write an entry to the raw.imfiles and raw.exposure tables + * set the state on the new.imfiles,new.exposure tables + * based on the camera config information; + * add an entry to the p1.pending table (mosaic) + - or + * add an entry to the p2.pending table (single) + +p0search -stats (expID) (camera): + * select a specified image in the new.imfiles/new.exposures table + * extract the specified header information + * search the summit metadata db tables + * report the image stats + [-update without output to MDDB] + +p0search -mkraw (expID) (camera): + * select a specified image in the new.imfiles/new.exposures table + * extract the specified header information + * search the summit metadata db tables + * write an entry to the raw.imfiles and raw.exposure tables + * set the state on the new.imfiles,new.exposure tables + +p0search -cleanup: + * remove completed entries from the new.imfiles,new.exposure tables + +** note : the division of -pending and -update allows separate processes + to be examining the image headers and measuring some stats. these + jobs can be run via pcontrol to reduce the load on the PanTasks + machines + Index: /branches/jh_branches/backtrack/ippTools/doc/p1tools.txt =================================================================== --- /branches/jh_branches/backtrack/ippTools/doc/p1tools.txt (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/doc/p1tools.txt (revision 22250) @@ -0,0 +1,16 @@ + +Phase 1 pipeline tools: + +p1search -define [constraints]: + * examine the raw.exposures tables and select exposures matching the given criteria + * add entries which are allowed (mosaic) to the p1.pending table + +p1search -pending : + * examine the p1.pending table and select exposures waiting for p1 + * output: lines consisting of: + (expID) (p1version) (camera) + +p1search -done : + * select completed entries in the p1.pending table + * move to the p1.done table + * add new entry to the p2.pending tables Index: /branches/jh_branches/backtrack/ippTools/doc/p2search.sh =================================================================== --- /branches/jh_branches/backtrack/ippTools/doc/p2search.sh (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/doc/p2search.sh (revision 22250) @@ -0,0 +1,8 @@ +#!/bin/csh -f + +echo "# expID class classID url" +echo "654320o fpa megacam 654320o.fits" +echo "654321o fpa megacam 654321o.fits" +echo "654322o fpa megacam 654322o.fits" +echo "654323o fpa megacam 654323o.fits" +echo "654324o fpa megacam 654324o.fits" Index: /branches/jh_branches/backtrack/ippTools/doc/p2tools.txt =================================================================== --- /branches/jh_branches/backtrack/ippTools/doc/p2tools.txt (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/doc/p2tools.txt (revision 22250) @@ -0,0 +1,41 @@ + +Phase 2 pipeline tools: + +p2search -quick + * search for images which match in raw.exp,raw.imfiles + * output in format which can be used by ppImage pantasks script + +p2search -define [options] + * input: searches mddb:raw_exposures,raw_images + - compares against p2Pending[Exp|Imfile] + - compares against p2Done[Exp|Imfile] + - XXX should be implemented as a special wrapper function that + implements this comparison as a SQL query + * output: updates mddb:P2_exposures_pending,P2_images_pending + * alternative output: identical to p2pending + +p2search -pending + * input: searches mddb:P2_exposures_pending,P2_images_pending + * output: Nlines consisting of: + (URL) (expID) (class) + * options: ? + +p2search -update + * examine the imfiles and identify any completed exposures + +p2search -done + * get exp_id/class/class_id from the CLI + * add the completed imfile to the p2DoneImfile tables + * remove corresponding entries from the p2PendingImfile table + * check to see if any p2PendingExps have no associated p2PendingImfiles + if so move the p2PendingExp(s) to p2DoneExp +// * send new entry to the pending p3 table + +ppImage file://path/filename file://path/outroot -recipe (recipe) +ppImage neb://nebname neb://outroot -recipe (recipe) + +restriction options: + -time (start) (stop) + -camera (camera) + -region (ra,dec) (ra,dec) + Index: /branches/jh_branches/backtrack/ippTools/doc/p3tools.txt =================================================================== --- /branches/jh_branches/backtrack/ippTools/doc/p3tools.txt (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/doc/p3tools.txt (revision 22250) @@ -0,0 +1,22 @@ + +Phase 3 pipeline tools: + +p3search -define : + * examine the raw.exposures tables and select exposures matching the given criteria + * add entries which are allowed (mosaic) to the p3.pending table + +p3search -quick : + * examine the raw.exposures tables and select exposures matching the given criteria + * return list of entries for p3 processing + * output: lines consisting of: + (expID) (p3version) (camera) + +p3search -pending : + * examine the p3.pending exposures table and select exposures waiting for p3 + * return list of entries for p3 processing + * output: lines consisting of: + (expID) (p3version) (camera) + +p3search -done : + * select completed entries in the p3.pending table + * move to the p3.done table Index: /branches/jh_branches/backtrack/ippTools/doc/summit_copy.txt =================================================================== --- /branches/jh_branches/backtrack/ippTools/doc/summit_copy.txt (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/doc/summit_copy.txt (revision 22250) @@ -0,0 +1,49 @@ +The summit copy phase (phase Z or pz) consists of three basic steps. +Discovering what exposures and available, discovering what image files are +associated with those exposures, and the downloading of image files. All +"external" communication with the DataStore is done with three utilities; +pzgetexp, pzgetimfiles & dsget. While local state information is accessed and +updated via the 'pztool' utility. + +The basic work follow is: + +pzgetexp retrieves a list of all of the exposures (filesets) available from a +DataStore "product". If any previously unknown file exposures are found they +are added to the pzPendingExp tables and the summitExp table. New entries to +the summitExp table have a imfile value of NULL as this information can't yet +be known. If any exposures are already in the summitExp table the most recent +exp_id is used to constrain the result set requested from the DataStore. + + ./pzgetexp -uri http://otis/ds/skyprobe/ -inst skyprobe -telescope ps1 + +The results of pzgetexp and accessible with pztool -pendingexp`. Which will +return a list of new exposures from the pzPendingExp table. + + ./pztool -pendingexp + +The output of which is used to invoke pzgetimfiles. + +pzgetimfiles retrieves a list of all of the imfiles (files) in an exposure +(fileset) from the DataStore. It then updates the imfiles count in the +summitExp table and adds each imfile to the pzPendingImfile table. + +./pzgetimfiles -uri http://otis/ds/skyprobe/sep8_twi_05/ -filesetid sep8_twi_05 -inst skyprobe -telescope ps1 + +pztool -pendingimfile returns the list of imfiles that need to be downloaded. + + ./pztool -pendingimfile + +The results of which are used to invoke dsget and pztool -copydone after dsget +completes it's download. + +dsget from the 'DataStore' package takes the follow parameters. + + dsget --uri --filename [--bytes ] [--md5 ] + +Where "filename" may be either just a filename relative to the CWD or a fully +qualified path. After dsget has copied the file pztool -copydone needs to be +invoked + + ./pztool -copydone -exp_id sep8_twi_03 -inst skyprobe -telescope ps1 -class chip -class_id sep8_twi_03.fits -uri file///data/alala/sep8_twi_03.fits + +And then we're Pau. Index: /branches/jh_branches/backtrack/ippTools/m4/ac_prog_perl_modules.m4 =================================================================== --- /branches/jh_branches/backtrack/ippTools/m4/ac_prog_perl_modules.m4 (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/m4/ac_prog_perl_modules.m4 (revision 22250) @@ -0,0 +1,53 @@ +dnl @synopsis AC_PROG_PERL_MODULES([MODULES], [ACTION-IF-TRUE], [ACTION-IF-FALSE]) +dnl +dnl Checks to see if the the given perl modules are available. If true +dnl the shell commands in ACTION-IF-TRUE are executed. If not the shell +dnl commands in ACTION-IF-FALSE are run. Note if $PERL is not set (for +dnl example by calling AC_CHECK_PROG, or AC_PATH_PROG), +dnl AC_CHECK_PROG(PERL, perl, perl) will be run. +dnl +dnl Example: +dnl +dnl AC_CHECK_PERL_MODULES(Text::Wrap Net::LDAP, , +dnl AC_MSG_WARN(Need some Perl modules) +dnl +dnl @category InstalledPackages +dnl @author Dean Povey +dnl @version 2002-09-25 +dnl @license AllPermissive + +AC_DEFUN([AC_PROG_PERL_MODULES],[ + ac_perl_modules="$1" + # Make sure we have perl + if test -z "$PERL"; then + AC_CHECK_PROG(PERL,perl,perl) + fi + + if test "x$PERL" != x; then + ac_perl_modules_failed=0 + for ac_perl_module in $ac_perl_modules; do + AC_MSG_CHECKING(for perl module $ac_perl_module) + + # Would be nice to log result here, but can't rely on autoconf + # internals + $PERL "-M$ac_perl_module" -e exit > /dev/null 2>&1 + if test $? -ne 0; then + AC_MSG_RESULT(no); + ac_perl_modules_failed=1 + else + AC_MSG_RESULT(ok); + fi + done + + # Run optional shell commands + if test "$ac_perl_modules_failed" = 0; then + : + $2 + else + : + $3 + fi + else + AC_MSG_WARN(could not find perl) + fi +]) Index: /branches/jh_branches/backtrack/ippTools/pxtools.pc.in =================================================================== --- /branches/jh_branches/backtrack/ippTools/pxtools.pc.in (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/pxtools.pc.in (revision 22250) @@ -0,0 +1,11 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@/@PACKAGE_NAME@ + +Name: @PACKAGE_NAME@ +Description: Pan-STARRS IPP Tools Library +Version: @VERSION@ +Requires: pslib psmodules ippdb +Libs: -L${libdir} -lpxtools +Cflags: -I${includedir} @PXTOOLS_CFLAGS@ Index: /branches/jh_branches/backtrack/ippTools/scripts/.cvsignore =================================================================== --- /branches/jh_branches/backtrack/ippTools/scripts/.cvsignore (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/scripts/.cvsignore (revision 22250) @@ -0,0 +1,2 @@ +Makefile +Makefile.in Index: /branches/jh_branches/backtrack/ippTools/scripts/camtest.sh =================================================================== --- /branches/jh_branches/backtrack/ippTools/scripts/camtest.sh (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/scripts/camtest.sh (revision 22250) @@ -0,0 +1,11 @@ +#!//bin/sh + +set -o verbose + +chiptest.sh || exit 1 + +camtool -pendingexp || exit 1 +camtool -pendingimfile || exit 1 + +camtool -addprocessedexp -cam_id 1 -uri file:///cam -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -sigma_ra 1 -sigma_dec 2 -zp_mean 10 -zp_stdev 2 -fwhm 42 -fwhm_range 100 -n_stars 2 -n_extended 0 -n_astrom 42 -n_cr 10000000 -path_base file:///foo || exit 1 +camtool -addprocessedexp -cam_id 2 -uri file:///cam -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -sigma_ra 1 -sigma_dec 2 -zp_mean 10 -zp_stdev 2 -fwhm 42 -fwhm_range 100 -n_stars 2 -n_extended 0 -n_astrom 42 -n_cr 10000000 -path_base file:///foo || exit 1 Index: /branches/jh_branches/backtrack/ippTools/scripts/chiptest.sh =================================================================== --- /branches/jh_branches/backtrack/ippTools/scripts/chiptest.sh (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/scripts/chiptest.sh (revision 22250) @@ -0,0 +1,17 @@ +#!//bin/sh + +set -o verbose + +regtest.sh || exit 1 + +chiptool -pendingimfile || exit 1 + +for ID in `seq 0 3`; do + chiptool -addprocessedimfile -chip_id 1 -exp_tag t10.1 -class_id $ID -uri file://chipp-t10.$ID -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file:///foo || exit 1 +done; + +for ID in `seq 0 3`; do + chiptool -addprocessedimfile -chip_id 2 -exp_tag t11.2 -class_id $ID -uri file://chipp-t11.$ID -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file:///foo || exit 1 +done; + +chiptool -pendingimfile || exit 1 Index: /branches/jh_branches/backtrack/ippTools/scripts/dettest.sh =================================================================== --- /branches/jh_branches/backtrack/ippTools/scripts/dettest.sh (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/scripts/dettest.sh (revision 22250) @@ -0,0 +1,65 @@ +#!//bin/sh + +set -o verbose + +det_id=1 + +./regtest.sh -detrend || exit 1 + +det_id=`dettool -definebyquery -det_type bias -filelevel fpa -select_exp_type bias -airmass_min 1 -airmass_max 10 -exp_time_min 10 -exp_time_max 30.0 -workdir file::///some/path -simple | cut -f1 -d" "` || exit 1 + +dettool -raw || exit 1 + +for ID in `seq 0 3` ; do + dettool -addprocessedimfile -det_id $det_id -exp_tag t10.1 -class_id $ID -uri file://proc-$ID -recip myrecip -bg 1 -bg_stdev 1 -bg_mean_stdev 1 || exit 1 +done; + +for ID in `seq 0 3` ; do + dettool -addprocessedimfile -det_id $det_id -exp_tag t11.2 -class_id $ID -uri file://proc-$ID -recip myrecip -bg 1 -bg_stdev 1 -bg_mean_stdev 1 || exit 1 +done; + + +dettool -tostacked || exit 1 + +for ID in `seq 0 3` ; do + dettool -addstacked -det_id $det_id -uri file://stacked-$ID -class_id $ID -recip myrecipe -bg 1 -bg_stdev 2 -bg_mean_stdev 3 || exit 1 +done; + +dettool -tonormalizedstat || exit 1 + +for ID in `seq 0 3` ; do + dettool -addnormalizedstat -det_id $det_id -class_id $ID -norm 0.12345 || exit 1 +done; + +dettool -tonormalize || exit 1 +for ID in `seq 0 3` ; do + dettool -addnormalizedimfile -det_id $det_id -class_id $ID -uri file://normalized-$ID -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base banana1 || exit 1 +done; + +dettool -tonormalizedexp || exit 1 +dettool -addnormalizedexp -det_id $det_id -recip myrecipe -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file://normalizedexp || exit 1 + +dettool -toresidimfile || exit 1 + +for ID in `seq 0 3` ; do + dettool -addresidimfile -det_id $det_id -exp_tag t10.1 -class_id $ID -recip myrecip -bg 1 -bg_stdev 1 -bg_mean_stdev 1 -uri file://resid-$ID || exit 1 +done; + +for ID in `seq 0 3` ; do + dettool -addresidimfile -det_id $det_id -exp_tag t11.2 -class_id $ID -recip myrecip -bg 1 -bg_stdev 1 -bg_mean_stdev 1 -uri file://resid-$ID || exit 1 +done; + +dettool -toresidexp || exit 1 +dettool -addresidexp -det_id $det_id -exp_tag t10.1 -recip myrecipe -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base jpeg1 || exit 1 +dettool -addresidexp -det_id $det_id -exp_tag t11.2 -recip myrecipe -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base jpeg1 -reject || exit 1 + +dettool -residdetrun || exit 1 +dettool -residexp || exit 1 +dettool -updateresidexp -det_id $det_id -iteration 0 -recip yourrecipe || exit 1 +dettool -updateresidexp -det_id $det_id -iteration 0 -exp_tag t11.2 -reject || exit 1 +dettool -updateresidexp -det_id $det_id -iteration 0 -exp_tag t11.2 || exit 1 + +dettool -adddetrunsummary -det_id $det_id -iteration 0 -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -accept || exit 1 +dettool -residexp || exit 1 + +dettool -updatedetrun -det_id $det_id -state stop || exit 1 Index: /branches/jh_branches/backtrack/ippTools/scripts/difftest.sh =================================================================== --- /branches/jh_branches/backtrack/ippTools/scripts/difftest.sh (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/scripts/difftest.sh (revision 22250) @@ -0,0 +1,16 @@ +#!//bin/sh + +set -o verbose + +warptest.sh || exit 1 + + +difftool -definerun -workdir file:///tmp/diff -skycell_id foo -tess_id bar || exit 1 +difftool -addinputskyfile -diff_id 1 -warp_id 1 -skycell_id foo1 -tess_id bar -kind warped -template || exit 1 +difftool -addinputskyfile -diff_id 1 -warp_id 1 -skycell_id foo2 -tess_id bar -kind warped || exit 1 +difftool -updaterun -state run -diff_id 1 || exit 1 +difftool -todiffskyfile || exit 1 +difftool -inputskyfile || exit 1 +difftool -adddiffskyfile -diff_id 1 -uri file:///tmp/diff/skyfile -path_base file://lalaland -bg 1 -bg_stdev 2 || exit 1 +difftool -diffskyfile -diff_id 1 || exit 1 +difftool -updaterun -state stop -diff_id 1 || exit 1 Index: /branches/jh_branches/backtrack/ippTools/scripts/mapfile.txt =================================================================== --- /branches/jh_branches/backtrack/ippTools/scripts/mapfile.txt (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/scripts/mapfile.txt (revision 22250) @@ -0,0 +1,19 @@ +warpSkyCellMap MULTI + +warpSkyCellMap METADATA + warp_id S64 1 + cam_id S64 1 + skycell_id STR foo1 + tess_id STR bar + class_id STR quix1 + fault S16 0 +END + +warpSkyCellMap METADATA + warp_id S64 1 + cam_id S64 1 + skycell_id STR foo2 + tess_id STR bar + class_id STR quix2 + fault S16 0 +END Index: /branches/jh_branches/backtrack/ippTools/scripts/regtest.sh =================================================================== --- /branches/jh_branches/backtrack/ippTools/scripts/regtest.sh (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/scripts/regtest.sh (revision 22250) @@ -0,0 +1,44 @@ +#!//bin/sh + +set -o verbose + +inject="pxinject" +regtool="regtool" + +echo -e "YES\nipp\n\n" | pxadmin -recreate || exit 1 + +exp_tag1=`$inject -newExp -exp_id t10 -inst gpc -telescope ps1 -imfiles 4 -workdir file::///some/path -simple` || exit 1 + +for ID in `seq 0 3`; do + $inject -newImfile -exp_tag $exp_tag1 -class_id $ID -uri file://$ID || exit 1 +done; + +$inject -updatenewExp -exp_tag $exp_tag1 -state run + +exp_tag2=`$inject -newExp -exp_id t11 -inst gpc -telescope ps1 -imfiles 4 -workdir file::///some/path -simple` || exit 1 + +for ID in `seq 0 3`; do + $inject -newImfile -exp_tag $exp_tag2 -class_id $ID -uri file://$ID || exit 1 +done; + +$inject -updatenewExp -exp_tag $exp_tag2 -state run + +$regtool -pendingimfile || exit 1 + +for ID in `seq 0 3`; do + $regtool -addprocessedimfile -exp_tag $exp_tag1 -class_id $ID -filter r -airmass 10 -ra 1 -decl 2 -exp_time 0 -bg 1 -bg_stdev 1 -bg_mean_stdev 10 -alt 10 -az 10 -ccd_temp 10 -posang 10 -object dog -dateobs 2006-10-20T10:10:10Z || exit 1 +done; + +$regtool -pendingexp|| exit 1 + +$regtool -addprocessedexp -exp_tag $exp_tag1 -filelevel OTA -filter r -airmass 10 -ra 1 -decl 2 -exp_type bias -exp_time 0 -bg 10 -bg_stdev 1 -bg_mean_stdev 10 -alt 10 -az 10 -ccd_temp 45 -posang 10 -object dog -dateobs "2006-10-20T10:10:10Z" -label foobar $* || exit 1 + +$regtool -pendingimfile || exit 1 + +for ID in `seq 0 3`; do + $regtool -addprocessedimfile -exp_tag $exp_tag2 -exp_type object -class_id $ID -filter r -airmass 10 -ra 1 -decl 2 -exp_time 0 -bg 1 -bg_stdev 1 -bg_mean_stdev 10 -alt 10 -az 10 -ccd_temp 10 -posang 10 -object dog -dateobs 2006-10-20T10:10:10Z || exit 1 +done; + +$regtool -pendingexp|| exit 1 + +$regtool -addprocessedexp -exp_tag $exp_tag2 -filelevel OTA -filter r -airmass 11 -ra 1 -decl 2 -exp_type bias -exp_time 0 -bg 11 -bg_stdev 1 -bg_mean_stdev 11 -alt 11 -az 11 -ccd_temp 45 -posang 11 -object dog -dateobs 2006-10-20T10:10:10Z -label foobar $* || exit 1 Index: /branches/jh_branches/backtrack/ippTools/scripts/warptest.sh =================================================================== --- /branches/jh_branches/backtrack/ippTools/scripts/warptest.sh (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/scripts/warptest.sh (revision 22250) @@ -0,0 +1,31 @@ +#!//bin/sh + +set -o verbose + +camtest.sh || exit 1 + +warptool -definerun -mode warp -workdir file:///foo || exit 1 + +warptool -addinputexp -warp_id 1 -cam_id 1 || exit 1 +warptool -addinputexp -warp_id 1 -cam_id 2 || exit 1 + +warptool -updaterun -warp_id 1 -state run || exit 1 + +warptool -exp -warp_id 1 || exit 1 + +warptool -imfile -warp_id 1 || exit 1 + +warptool -tooverlap -warp_id 1 || exit 1 + +warptool -addoverlap -mapfile mapfile.txt || exit 1 + +warptool -scmap || exit 1 + +warptool -towarped || exit 1 + +warptool -addwarped -warp_id 1 -skycell_id foo1 -tess_id bar -uri file:///tmp/foo -path_base file://wonderland -bg 1 -bg_stdev 2 || exit 1 +warptool -addwarped -warp_id 1 -skycell_id foo2 -tess_id bar -uri file:///tmp/foo -path_base file://wonderland -bg 1 -bg_stdev 2 || exit 1 + +warptool -warped -warp_id 1 || exit 1 + +#warptool -updaterun -warp_id 1 -state stop || exit 1 Index: /branches/jh_branches/backtrack/ippTools/share/.cvsignore =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/.cvsignore (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/.cvsignore (revision 22250) @@ -0,0 +1,2 @@ +Makefile.in +Makefile Index: /branches/jh_branches/backtrack/ippTools/share/Makefile.am =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/Makefile.am (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/Makefile.am (revision 22250) @@ -0,0 +1,33 @@ +dist_pkgdata_DATA = \ + camtool_find_pendingexp.sql \ + camtool_find_pendingimfile.sql \ + camtool_find_processedexp.sql \ + camtool_queue_chip_id.sql \ + chiptool_completely_processed_exp.sql \ + chiptool_find_rawexp.sql \ + chiptool_find_unprocessed_imfile.pl \ + chiptool_pendingimfile.sql \ + chiptool_processedimfile.sql \ + chiptool_queuerawexp.sql \ + chiptool_queuerawimfile.sql \ + detselect_search.sql \ + detselect_select.sql \ + dettool_find_completed_runs.sql \ + difftool_inputskyfile.sql \ + difftool_skyfile.sql \ + difftool_todiffskyfile.sql \ + regtool_find_unprocessed_exp.sql \ + regtool_find_unprocessed_imfile.sql \ + regtool_pendingexp.sql \ + regtool_pendingimfile.sql \ + regtool_processedexp.sql \ + regtool_processedimfile.sql \ + stacktool_inputskyfile.sql \ + stacktool_sumskyfile.sql \ + stacktool_tosum.sql \ + warptool_exp.sql \ + warptool_imfile.sql \ + warptool_scmap.sql \ + warptool_tooverlap.sql \ + warptool_towarped.sql \ + warptool_warped.sql Index: /branches/jh_branches/backtrack/ippTools/share/camtool_find_chip_id.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/camtool_find_chip_id.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/camtool_find_chip_id.sql (revision 22250) @@ -0,0 +1,13 @@ +-- this query is used by both camtool -pendingexp & camtool -addprocessedexp it +-- does a little more work then is necessary for -addprocessed but it seems +-- "cleaner" to use the same query for both cases +SELECT + * +FROM + (SELECT DISTINCT + chipRun.* + FROM chipRun + JOIN chipProcessedExp + USING(chip_id) + WHERE + chipRun.state = 'stop') as Foo Index: /branches/jh_branches/backtrack/ippTools/share/camtool_find_pendingexp.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/camtool_find_pendingexp.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/camtool_find_pendingexp.sql (revision 22250) @@ -0,0 +1,26 @@ +-- this query is used by both camtool -pendingexp & camtool -addprocessedexp it +-- does a little more work then is necessary for -addprocessed but it seems +-- "cleaner" to use the same query for both cases +SELECT + camRun.*, + rawExp.exp_tag, + rawExp.exp_id, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel +FROM camRun +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + USING(chip_id) +JOIN rawExp + ON chipProcessedImfile.exp_tag = rawExp.exp_tag +LEFT JOIN camProcessedExp + ON camRun.cam_id = camProcessedExp.cam_id +LEFT JOIN camMask + ON camRun.label = camMask.label +WHERE + chipRun.state = 'stop' + AND camRun.state = 'run' + AND camMask.label IS NULL + AND camProcessedExp.cam_id IS NULL Index: /branches/jh_branches/backtrack/ippTools/share/camtool_find_pendingimfile.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/camtool_find_pendingimfile.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/camtool_find_pendingimfile.sql (revision 22250) @@ -0,0 +1,31 @@ +SELECT * FROM + -- the subselect is so where criteria can be specified without knowing + -- which table the field came from + (SELECT + camPendingExp.*, + chipProcessedExp.exp_tag, + chipProcessedExp.guide_id, + chipProcessedImfile.class_id, + chipProcessedImfile.uri, + chipProcessedImfile.bg, + chipProcessedImfile.bg_stdev, + chipProcessedImfile.bg_mean_stdev, + chipProcessedImfile.path_base, + rawExp.exp_id, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel + FROM camPendingExp + JOIN chipProcessedExp + USING(chip_id) + JOIN chipProcessedImfile + USING(chip_id) + JOIN rawExp + ON chipProcessedExp.exp_tag = rawExp.exp_tag + LEFT JOIN camProcessedExp + ON camPendingExp.cam_id = camProcessedExp.cam_id + LEFT JOIN camMask + ON camPendingExp.label = camMask.label + WHERE + camProcessedExp.cam_id IS NULL + AND camMask.label IS NULL) as foo Index: /branches/jh_branches/backtrack/ippTools/share/camtool_find_processedexp.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/camtool_find_processedexp.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/camtool_find_processedexp.sql (revision 22250) @@ -0,0 +1,13 @@ +SELECT + camProcessedExp.*, + rawExp.exp_id, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel +FROM camProcessedExp +JOIN chipProcessedExp + USING(chip_id) +JOIN rawExp + ON chipProcessedExp.exp_tag = rawExp.exp_tag +WHERE -- bogus condition so there is a pre-existing where to append to + camProcessedExp.cam_id IS NOT NULL Index: /branches/jh_branches/backtrack/ippTools/share/camtool_queue_chip_id.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/camtool_queue_chip_id.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/camtool_queue_chip_id.sql (revision 22250) @@ -0,0 +1,17 @@ +-- camtool only operates on exposures so we can safely queue more then one at a +-- time without worrying about losing the track of the generated cam_id +INSERT INTO camRun + SElECT + 0, -- cam_id + chip_id, -- chip_id + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- label + '%s', -- reduction + '%s', -- expgroup + '%s' -- dvodb + FROM chipRun + WHERE + chipRun.state = 'stop' + AND chipRun.chip_id = %lld Index: /branches/jh_branches/backtrack/ippTools/share/chiptool_completely_processed_exp.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/chiptool_completely_processed_exp.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/chiptool_completely_processed_exp.sql (revision 22250) @@ -0,0 +1,31 @@ +-- the output of this query must match the format of chipRun row +SELECT DISTINCT + chip_id, + state, + workdir, + workdir_state, + label, + reduction, + expgroup, + dvodb +FROM + (SELECT + chipRun.*, + rawExp.imfiles, + chipProcessedImfile.class_id + FROM chipRun + JOIN chipInputImfile + USING(chip_id) + JOIN rawExp + ON chipInputImfile.exp_tag = rawExp.exp_tag + JOIN chipProcessedImfile + ON chipInputImfile.chip_id = chipProcessedImfile.chip_id + AND chipInputImfile.exp_tag = chipProcessedImfile.exp_tag + AND chipInputImfile.class_id = chipProcessedImfile.class_id + WHERE + chipRun.state = 'run' + AND chipProcessedImfile.fault = 0 + GROUP BY + chipInputImfile.chip_id, + chipInputImfile.exp_tag + HAVING rawExp.imfiles = COUNT(chipProcessedImfile.class_id)) as Foo Index: /branches/jh_branches/backtrack/ippTools/share/chiptool_find_rawexp.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/chiptool_find_rawexp.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/chiptool_find_rawexp.sql (revision 22250) @@ -0,0 +1,7 @@ +-- this query is used to find potental rawExps to be queued for chiptool +-- processeing +SELECT + exp_tag +FROM rawExp +WHERE + rawExp.fault = 0 Index: /branches/jh_branches/backtrack/ippTools/share/chiptool_find_unprocessed_imfile.pl =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/chiptool_find_unprocessed_imfile.pl (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/chiptool_find_unprocessed_imfile.pl (revision 22250) @@ -0,0 +1,8 @@ +SELECT DISTINCT + chipInputImfile.* +FROM chipInputImfile +LEFT JOIN chipProcessedImfile + USING(chip_id, class_id) +WHERE + chipProcessedImfile.chip_id IS NULL + AND chipProcessedImfile.class_id IS NULL Index: /branches/jh_branches/backtrack/ippTools/share/chiptool_pendingimfile.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/chiptool_pendingimfile.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/chiptool_pendingimfile.sql (revision 22250) @@ -0,0 +1,29 @@ +SELECT + chipRun.*, + chipInputImfile.exp_tag, + chipInputImfile.class_id, + rawImfile.uri, + rawExp.exp_id, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel +FROM chipRun +JOIN chipInputImfile + USING(chip_id) +LEFT JOIN chipProcessedImfile + ON chipInputImfile.chip_id = chipProcessedImfile.chip_id + AND chipInputImfile.exp_tag = chipProcessedImfile.exp_tag + AND chipInputImfile.class_id = chipProcessedImfile.class_id +JOIN rawImfile + ON chipInputImfile.exp_tag = rawImfile.exp_tag + AND chipInputImfile.class_id = rawImfile.class_id +JOIN rawExp + ON rawImfile.exp_tag = rawExp.exp_tag +LEFT JOIN chipMask + ON chipRun.label = chipMask.label +WHERE + chipRun.state = 'run' + AND chipProcessedImfile.chip_id IS NULL + AND chipProcessedImfile.exp_tag IS NULL + AND chipProcessedImfile.class_id IS NULL + AND chipMask.label IS NULL Index: /branches/jh_branches/backtrack/ippTools/share/chiptool_processedimfile.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/chiptool_processedimfile.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/chiptool_processedimfile.sql (revision 22250) @@ -0,0 +1,21 @@ +SELECT + chipProcessedExp.*, + chipProcessedImfile.class_id, + chipProcessedImfile.uri, + chipProcessedImfile.bg, + chipProcessedImfile.bg_stdev, + chipProcessedImfile.bg_mean_stdev, + chipProcessedImfile.path_base, + rawExp.exp_id, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel +FROM chipProcessedImfile +JOIN chipProcessedExp + USING(chip_id) +JOIN rawExp + ON chipProcessedExp.exp_tag = rawExp.exp_tag +WHERE +-- bogus test; just here so there there is a 'WHERE' stmt to append conditionals too + chipProcessedImfile.exp_tag is NOT NULL + Index: /branches/jh_branches/backtrack/ippTools/share/chiptool_queuerawimfile.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/chiptool_queuerawimfile.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/chiptool_queuerawimfile.sql (revision 22250) @@ -0,0 +1,13 @@ +INSERT INTO chipInputImfile + SELECT + LAST_INSERT_ID(), + rawImfile.exp_tag, + rawImfile.class_id + FROM rawImfile + JOIN newExp + USING(exp_tag) + WHERE + rawImfile.fault = 0 + AND newExp.state = 'stop' + AND rawImfile.exp_tag = '%s' + Index: /branches/jh_branches/backtrack/ippTools/share/detselect_search.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/detselect_search.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/detselect_search.sql (revision 22250) @@ -0,0 +1,11 @@ +SELECT DISTINCT + detRunSummary.det_id, + detRunSummary.iteration, + detRun.filelevel +FROM detRun +JOIN detRunSummary + USING(det_id) +WHERE + detRun.state = 'stop' + AND detRun.mode = 'master' + AND detRunSummary.accept = 1 Index: /branches/jh_branches/backtrack/ippTools/share/detselect_select.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/detselect_select.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/detselect_select.sql (revision 22250) @@ -0,0 +1,12 @@ +SELECT DISTINCT + detNormalizedImfile.* +FROM detNormalizedImfile +JOIN detRun + USING(det_id) +JOIN detRunSummary + ON detNormalizedImfile.det_id = detRunSummary.det_id + AND detNormalizedImfile.iteration = detRunSummary.iteration +WHERE + detRun.state = 'stop' + AND detRun.mode = 'master' + AND detRunSummary.accept = 1 Index: /branches/jh_branches/backtrack/ippTools/share/dettool_find_completed_runs.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/dettool_find_completed_runs.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/dettool_find_completed_runs.sql (revision 22250) @@ -0,0 +1,26 @@ +SELECT DISTINCT + det_id, + iteration +FROM + (SELECT DISTINCT + detRun.det_id, + detRun.iteration, + detInputExp.exp_tag + FROM detRun + LEFT JOIN detInputExp + ON detRun.det_id = detInputExp.det_id + AND detRun.iteration = detInputExp.iteration + LEFT JOIN rawExp + ON detInputExp.exp_tag = rawExp.exp_tag + LEFT JOIN detResidExp + ON detRun.det_id = detResidExp.det_id + AND detRun.iteration = detResidExp.iteration + AND detInputExp.exp_tag = detResidExp.exp_tag + WHERE + detRun.state = 'run' + GROUP BY + detRun.det_id, + detRun.iteration + HAVING + COUNT(detResidExp.exp_tag) = COUNT(detInputExp.exp_tag) + ) AS residdetrun Index: /branches/jh_branches/backtrack/ippTools/share/difftool_inputskyfile.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/difftool_inputskyfile.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/difftool_inputskyfile.sql (revision 22250) @@ -0,0 +1,22 @@ +SELECT + warpSkyfile.*, + diffInputSkyfile.template, + rawExp.camera +FROM diffRun +JOIN diffInputSkyfile + USING(diff_id) +JOIN warpSkyfile + ON diffInputSkyfile.warp_id = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.tess_id = warpSkyfile.tess_id +JOIN warpInputExp + ON diffInputSkyfile.warp_id = warpInputExp.warp_id +JOIN camProcessedExp + ON warpInputExp.cam_id = camProcessedExp.cam_id +JOIN chipProcessedExp + ON camProcessedExp.chip_id = chipProcessedExp.chip_id +JOIN rawExp + ON chipProcessedExp.exp_tag = rawExp.exp_tag +WHERE + diffRun.state = 'run' + Index: /branches/jh_branches/backtrack/ippTools/share/difftool_skyfile.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/difftool_skyfile.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/difftool_skyfile.sql (revision 22250) @@ -0,0 +1,10 @@ +SELECT + diffRun.skycell_id, + diffRun.tess_id, + diffSkyfile.* +FROM diffRun +JOIN diffSkyfile + USING(diff_id) +WHERE + diffRun.state = 'run' + Index: /branches/jh_branches/backtrack/ippTools/share/difftool_todiffskyfile.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/difftool_todiffskyfile.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/difftool_todiffskyfile.sql (revision 22250) @@ -0,0 +1,16 @@ +SELECT DISTINCT + diffRun.diff_id, + diffRun.workdir +FROM diffRun +JOIN diffInputSkyfile + USING(diff_id) +JOIN warpSkyfile + ON diffInputSkyfile.warp_id = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.tess_id = warpSkyfile.tess_id +LEFT JOIN diffSkyfile + ON diffInputSkyfile.diff_id = diffSkyfile.diff_id +WHERE + diffRun.state = 'run' + AND diffSkyfile.diff_id IS NULL + Index: /branches/jh_branches/backtrack/ippTools/share/regtool_find_unprocessed_exp.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/regtool_find_unprocessed_exp.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/regtool_find_unprocessed_exp.sql (revision 22250) @@ -0,0 +1,16 @@ +SELECT DISTINCT * FROM + (SELECT + newExp.* + FROM newExp + JOIN newImfile + USING(exp_tag) + LEFT JOIN rawExp + USING(exp_tag) + WHERE + newExp.state = 'run' + AND rawExp.exp_tag IS NULL + AND newExp.imfiles = + (SELECT COUNT(exp_tag) FROM rawImfile + WHERE rawImfile.exp_tag = newExp.exp_tag)) as Foo +WHERE + exp_tag = '%s' Index: /branches/jh_branches/backtrack/ippTools/share/regtool_find_unprocessed_imfile.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/regtool_find_unprocessed_imfile.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/regtool_find_unprocessed_imfile.sql (revision 22250) @@ -0,0 +1,11 @@ +SELECT DISTINCT * FROM + (SELECT + newImfile.* + FROM newImfile + JOIN newExp + USING(exp_tag) + LEFT JOIN rawExp + USING(exp_tag) + WHERE + newExp.state = 'run' + AND rawExp.exp_tag IS NULL) as Foo Index: /branches/jh_branches/backtrack/ippTools/share/regtool_pendingexp.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/regtool_pendingexp.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/regtool_pendingexp.sql (revision 22250) @@ -0,0 +1,15 @@ +SELECT + newExp.* +FROM newExp +JOIN newImfile + USING(exp_tag) +LEFT JOIN rawExp + USING(exp_tag) +WHERE + newExp.state = 'run' + AND rawExp.exp_tag IS NULL + AND newExp.imfiles = + (SELECT COUNT(exp_tag) FROM rawImfile + WHERE + rawImfile.exp_tag = newExp.exp_tag + AND rawImfile.fault = 0) Index: /branches/jh_branches/backtrack/ippTools/share/regtool_pendingimfile.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/regtool_pendingimfile.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/regtool_pendingimfile.sql (revision 22250) @@ -0,0 +1,11 @@ +SELECT + newImfile.*, + newExp.workdir +FROM newImfile +LEFT JOIN newExp + USING(exp_tag) +LEFT JOIN rawExp + USING(exp_tag) +WHERE + newExp.exp_tag is NOT NULL + AND rawExp.exp_tag IS NULL Index: /branches/jh_branches/backtrack/ippTools/share/regtool_processedimfile.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/regtool_processedimfile.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/regtool_processedimfile.sql (revision 22250) @@ -0,0 +1,10 @@ +SELECT + rawImfile.*, + newExp.exp_id, + newExp.camera, + newExp.telescope +FROM rawImfile +JOIN newExp + USING(exp_tag) +-- bogus conditional so there is a where clause to append to +WHERE rawImfile.exp_tag is NOT NULL Index: /branches/jh_branches/backtrack/ippTools/share/stacktool_inputskyfile.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/stacktool_inputskyfile.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/stacktool_inputskyfile.sql (revision 22250) @@ -0,0 +1,20 @@ + SELECT + warpSkyfile.*, + rawExp.camera +FROM stackRun +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpSkyfile + ON stackInputSkyfile.warp_id = warpSkyfile.warp_id + AND stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.tess_id = warpSkyfile.tess_id +JOIN warpInputExp + ON stackInputSkyfile.warp_id = warpInputExp.warp_id +JOIN camProcessedExp + ON warpInputExp.cam_id = camProcessedExp.cam_id +JOIN chipProcessedExp + ON camProcessedExp.chip_id = chipProcessedExp.chip_id +JOIN rawExp + ON chipProcessedExp.exp_tag = rawExp.exp_tag +WHERE + stackRun.state = 'run' Index: /branches/jh_branches/backtrack/ippTools/share/stacktool_sumskyfile.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/stacktool_sumskyfile.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/stacktool_sumskyfile.sql (revision 22250) @@ -0,0 +1,7 @@ +SELECT + stackSumSkyfile.* +FROM stackRun +JOIN stackSumSkyfile + USING(stack_id) +WHERE + stackRun.state = 'run' Index: /branches/jh_branches/backtrack/ippTools/share/stacktool_tosum.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/stacktool_tosum.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/stacktool_tosum.sql (revision 22250) @@ -0,0 +1,9 @@ +SELECT + stackRun.stack_id, + stackRun.workdir +FROM stackRun +LEFT JOIN stackSumSkyfile + USING(stack_id) +WHERE + stackRun.state = 'run' + AND stackSumSkyfile.stack_id IS NULL Index: /branches/jh_branches/backtrack/ippTools/share/warptool_exp.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/warptool_exp.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/warptool_exp.sql (revision 22250) @@ -0,0 +1,10 @@ +SELECT + camProcessedExp.* +FROM warpRun +JOIN warpInputExp + USING(warp_id) +JOIN camProcessedExp + ON warpInputExp.cam_id = camProcessedExp.cam_id +WHERE + warpRun.state = 'run' + AND camProcessedExp.fault = 0 Index: /branches/jh_branches/backtrack/ippTools/share/warptool_imfile.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/warptool_imfile.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/warptool_imfile.sql (revision 22250) @@ -0,0 +1,19 @@ +SELECT DISTINCT + rawImfile.*, + warpInputExp.cam_id, + chipProcessedImfile.path_base as chip_path_base, + camProcessedExp.path_base as cam_path_base +FROM warpRun +JOIN warpInputExp + USING(warp_id) +JOIN camProcessedExp + ON warpInputExp.cam_id = camProcessedExp.cam_id +JOIN chipProcessedExp + ON camProcessedExp.chip_id = chipProcessedExp.chip_id +JOIN chipProcessedImfile + ON camProcessedExp.chip_id = chipProcessedImfile.chip_id +JOIN rawImfile -- is there any reason not to refer back to rawimfiles? + ON chipProcessedExp.exp_tag = rawImfile.exp_tag +WHERE + warpRun.state = 'run' + Index: /branches/jh_branches/backtrack/ippTools/share/warptool_scmap.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/warptool_scmap.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/warptool_scmap.sql (revision 22250) @@ -0,0 +1,16 @@ +SELECT + warpSkyCellMap.*, + chipProcessedImfile.uri, + chipProcessedImfile.path_base as chip_path_base, + camProcessedExp.path_base as cam_path_base +FROM warpRun +JOIN warpSkyCellMap + USING(warp_id) +JOIN camProcessedExp + ON warpSkyCellMap.cam_id = camProcessedExp.cam_id +JOIN chipProcessedImfile + ON camProcessedExp.chip_id = chipProcessedImfile.chip_id + AND warpSkyCellMap.class_id = chipProcessedImfile.class_id +WHERE + warpRun.state = 'run' + Index: /branches/jh_branches/backtrack/ippTools/share/warptool_tooverlap.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/warptool_tooverlap.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/warptool_tooverlap.sql (revision 22250) @@ -0,0 +1,22 @@ +SELECT + warpRun.warp_id, + camProcessedExp.cam_id, + warpRun.workdir, + rawExp.camera +FROM warpRun +JOIN warpInputExp + USING(warp_id) +JOIN camProcessedExp + ON warpInputExp.cam_id = camProcessedExp.cam_id +JOIN chipProcessedExp + ON camProcessedExp.chip_id = chipProcessedExp.chip_id +JOIN rawExp + ON chipProcessedExp.exp_tag = rawExp.exp_tag +LEFT JOIN warpSkyCellMap + ON warpInputExp.warp_id = warpSkyCellMap.warp_id + AND warpInputExp.cam_id = warpSkyCellMap.cam_id +WHERE + warpRun.state = 'run' + AND camProcessedExp.fault = 0 + AND warpSkyCellMap.warp_id IS NULL + AND warpSkyCellMap.cam_id IS NULL Index: /branches/jh_branches/backtrack/ippTools/share/warptool_towarped.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/warptool_towarped.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/warptool_towarped.sql (revision 22250) @@ -0,0 +1,26 @@ +SELECT DISTINCT + warpSkyCellMap.warp_id, + warpSkyCellMap.skycell_id, + warpSkyCellMap.tess_id, + warpSkyCellMap.cam_id, + rawExp.camera, + warpRun.workdir +FROM warpRun +JOIN warpSkyCellMap + USING(warp_id) +JOIN camProcessedExp + ON warpSkyCellMap.cam_id = camProcessedExp.cam_id +JOIN chipProcessedExp + ON camProcessedExp.chip_id = chipProcessedExp.chip_id +JOIN rawExp + ON chipProcessedExp.exp_tag = rawExp.exp_tag +LEFT JOIN warpSkyfile + ON warpRun.warp_id = warpSkyfile.warp_id + AND warpSkyCellMap.skycell_id = warpSkyfile.skycell_id + AND warpSkyCellMap.tess_id = warpSkyfile.tess_id +WHERE + warpRun.state = 'run' + AND warpSkyfile.warp_id IS NULL + AND warpSkyfile.skycell_id IS NULL + AND warpSkyfile.tess_id IS NULL + Index: /branches/jh_branches/backtrack/ippTools/share/warptool_warped.sql =================================================================== --- /branches/jh_branches/backtrack/ippTools/share/warptool_warped.sql (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/share/warptool_warped.sql (revision 22250) @@ -0,0 +1,7 @@ +SELECT + warpSkyfile.* +FROM warpRun +JOIN warpSkyfile + USING(warp_id) +WHERE + warpRun.state = 'run' Index: /branches/jh_branches/backtrack/ippTools/src/.cvsignore =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/.cvsignore (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/.cvsignore (revision 22250) @@ -0,0 +1,27 @@ +.deps +.gdb_history +.libs +Makefile +Makefile.in +config.h +config.h.in +stamp-h1 +*.la +*.lo +pxtoolsErrorCodes.c +pxtoolsErrorCodes.h +pxadmin +pxinject +pztool +pzgetexp +pzgetimfiles +regtool +guidetool +chiptool +camtool +warptool +difftool +stacktool +dettool +detselect +pxdata.c Index: /branches/jh_branches/backtrack/ippTools/src/Makefile.am =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/Makefile.am (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/Makefile.am (revision 22250) @@ -0,0 +1,170 @@ + +bin_PROGRAMS = \ + detselect \ + dettool \ + regtool \ + chiptool \ + camtool \ + warptool \ + difftool \ + stacktool \ + pxadmin \ + pxinject \ + pzgetexp \ + pzgetimfiles \ + pztool + +pkginclude_HEADERS = \ + pxtoolsErrorCodes.h \ + pxadmin.h \ + pxdata.h \ + pxio.h \ + pxtag.h \ + pxregister.h \ + pxchip.h \ + pxcam.h \ + pxconfig.h \ + pxtools.h + +noinst_HEADERS = \ + dettool.h \ + detselect.h \ + regtool.h \ + guidetool.h \ + chiptool.h \ + camtool.h \ + warptool.h \ + difftool.h \ + stacktool.h \ + pxinject.h \ + pzgetexp.h \ + pzgetimfiles.h \ + pztool.h + +lib_LTLIBRARIES = libpxtools.la +libpxtools_la_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +libpxtools_la_LIBS = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) +libpxtools_la_LDFLAGS = -release $(PACKAGE_VERSION) +libpxtools_la_SOURCES = \ + pxtoolsErrorCodes.c \ + pxerrors.c \ + pxconfig.c \ + pxfault.c \ + pxtables.c \ + pxdata.c \ + pxio.c \ + pxtag.c \ + pxregister.c \ + pxchip.c \ + pxcam.c + +# for pxtools.h +AM_CPPFLAGS = -I$(top_srcdir)/src$ + +pztool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pztool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pztool_SOURCES = \ + pztool.c \ + pztoolConfig.c + +regtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +regtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +regtool_SOURCES = \ + regtool.c \ + regtoolConfig.c + +guidetool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +guidetool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +guidetool_SOURCES = \ + guidetool.c \ + guidetoolConfig.c + +chiptool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +chiptool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +chiptool_SOURCES = \ + chiptool.c \ + chiptoolConfig.c + +camtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +camtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +camtool_SOURCES = \ + camtool.c \ + camtoolConfig.c + +warptool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +warptool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +warptool_SOURCES = \ + warptool.c \ + warptoolConfig.c + +difftool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +difftool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +difftool_SOURCES = \ + difftool.c \ + difftoolConfig.c + +stacktool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +stacktool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +stacktool_SOURCES = \ + stacktool.c \ + stacktoolConfig.c + +pxadmin_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pxadmin_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pxadmin_SOURCES = \ + pxadmin.c \ + pxadminConfig.c + +pzgetexp_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pzgetexp_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pzgetexp_SOURCES = \ + pzgetexp.c \ + pzgetexpConfig.c + +pzgetimfiles_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pzgetimfiles_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pzgetimfiles_SOURCES = \ + pzgetimfiles.c \ + pzgetimfilesConfig.c + +dettool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +dettool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +dettool_SOURCES = \ + dettool.c \ + dettoolConfig.c + +detselect_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +detselect_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +detselect_SOURCES = \ + detselect.c \ + detselectConfig.c + +pxinject_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pxinject_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pxinject_SOURCES = \ + pxinject.c \ + pxinjectConfig.c + +clean-local: + -rm -f TAGS +# Tags for emacs +tags: + etags `find . -name \*.[ch] -print` + +# Error codes. +BUILT_SOURCES = pxtoolsErrorCodes.h pxtoolsErrorCodes.c +CLEANFILES = pxtoolsErrorCodes.h pxtoolsErrorCodes.c +EXTRA_DIST = pxtoolsErrorCodes.dat pxtoolsErrorCodes.c.in pxtoolsErrorCodes.h.in + +pxtoolsErrorCodes.h : pxtoolsErrorCodes.dat pxtoolsErrorCodes.h.in + $(ERRORCODES) --data=pxtoolsErrorCodes.dat --outdir=. pxtoolsErrorCodes.h + +pxtoolsErrorCodes.c : pxtoolsErrorCodes.dat pxtoolsErrorCodes.c.in pxtoolsErrorCodes.h + $(ERRORCODES) --data=pxtoolsErrorCodes.dat --outdir=. pxtoolsErrorCodes.c + +BUILT_SOURCES += pxdata.c +EXTRA_DIST += pxdata.c.template + +pxdata.c: $(srcdir)/pxdata.c.template + $(PERL) -pe 's|PKGDATADIR|$(pkgdatadir)|' $? > $@ + Index: /branches/jh_branches/backtrack/ippTools/src/camtool.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/camtool.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/camtool.c (revision 22250) @@ -0,0 +1,884 @@ +/* + * camtool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "camtool.h" + +static bool queueMode(pxConfig *config); +static bool pendingexpMode(pxConfig *config); +static bool pendingimfileMode(pxConfig *config); +static bool addprocessedexpMode(pxConfig *config); +static bool processedexpMode(pxConfig *config); +static bool updateprocessedexpMode(pxConfig *config); +static bool blockMode(pxConfig *config); +static bool maskedMode(pxConfig *config); +static bool unblockMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = camtoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(CAMTOOL_MODE_QUEUE, queueMode); + MODECASE(CAMTOOL_MODE_PENDINGEXP, pendingexpMode); + MODECASE(CAMTOOL_MODE_PENDINGIMFILE, pendingimfileMode); + MODECASE(CAMTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpMode); + MODECASE(CAMTOOL_MODE_PROCESSEDEXP, processedexpMode); + MODECASE(CAMTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + MODECASE(CAMTOOL_MODE_BLOCK, blockMode); + MODECASE(CAMTOOL_MODE_MASKED, maskedMode); + MODECASE(CAMTOOL_MODE_UNBLOCK, unblockMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +#define ADDPARAMSTR(from, to, name) \ +{ \ + bool status = false; \ + psString str = psMetadataLookupStr(&status, from, "-" name); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -" name); \ + return false; \ + } \ + if (str) { \ + if (!psMetadataAddStr(to, PS_LIST_TAIL, name, 0, "==", str)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " name); \ + psFree(to); \ + return false; \ + } \ + } \ +} + + +#define ADDPARAMF(from, to, type, name) \ +{ \ + bool status = false; \ + ps##type var = psMetadataLookup##type(&status, from, "-" name); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -" name); \ + return false; \ + } \ + if (!isnan(var)) { \ + if (!psMetadataAdd##type(to, PS_LIST_TAIL, #name, 0, "==", var)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " name); \ + psFree(to); \ + return false; \ + } \ + } \ +} + +static bool queueMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); +{ + bool status = false; + psString var = psMetadataLookupStr(&status, config->args, "-chip_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -chip_id"); + return false; + } + if (var) { + if (!psMetadataAddS64(where, PS_LIST_TAIL, "chip_id", 0, "==", (psS64)atoll(var))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item chip_id"); + psFree(where); + return false; + } + } +} + + ADDPARAMSTR(config->args, where, "exp_tag"); + ADDPARAMSTR(config->args, where, "exp_id"); + ADDPARAMSTR(config->args, where, "inst"); + ADDPARAMSTR(config->args, where, "telescope"); + ADDPARAMSTR(config->args, where, "dateobs"); + ADDPARAMSTR(config->args, where, "exp_type"); + ADDPARAMSTR(config->args, where, "imfiles"); + ADDPARAMSTR(config->args, where, "workdir"); + ADDPARAMSTR(config->args, where, "filter"); + ADDPARAMF(config->args, where, F32, "airmass"); + ADDPARAMF(config->args, where, F32, "ra"); + ADDPARAMF(config->args, where, F32, "decl"); + ADDPARAMF(config->args, where, F32, "exp_time"); + ADDPARAMF(config->args, where, F64, "bg"); + ADDPARAMF(config->args, where, F64, "bg_stdev"); + ADDPARAMF(config->args, where, F64, "bg_mean_stdev"); + ADDPARAMF(config->args, where, F64, "alt"); + ADDPARAMF(config->args, where, F64, "az"); + ADDPARAMF(config->args, where, F32, "ccd_temp"); + ADDPARAMF(config->args, where, F64, "posang"); + ADDPARAMSTR(config->args, where, "object"); + + if (where->list->n < 1) { + psFree(where); + where = NULL; + } + + bool status = false; + psString workdir = psMetadataLookupStr(&status, config->args, "-set_workdir"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_workdir"); + return false; + } + + psString label = psMetadataLookupStr(&status, config->args, "-set_label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_label"); + return false; + } + + psString reduction = psMetadataLookupStr(&status, config->args, "-set_reduction"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_reduction"); + return false; + } + + psString expgroup = psMetadataLookupStr(&status, config->args, "-set_expgroup"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_expgroup"); + return false; + } + + psString dvodb = psMetadataLookupStr(&status, config->args, "-set_dvodb"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_dvodb"); + return false; + } + + // find the chipProcessedExp exposures that we want to queue up. +// psString query = psStringCopy("SELECT * FROM chipRun WHERE chipRun.state = 'stop'"); + psString query = pxDataGet("camtool_find_chip_id.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, ""); + psFree(where); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // would could do this "all in the database" if we didn't want the option + // of changing the label/reduction/expgroup/dvodb/etc. So we're pulling the + // data out so we have the option of changing these values or leaving the + // old values in place (i.e., passing the values through). + + // loop over our list of chipProcessedExp rows + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + chipRunRow *row = chipRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxcamQueueByChipID(config, + row->chip_id, + workdir ? workdir : row->workdir, + label ? label : row->label, + reduction ? reduction : row->reduction, + expgroup ? expgroup : row->expgroup, + dvodb ? dvodb : row->dvodb + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue chip_id: %" PRId64, row->chip_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool pendingexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("camtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "camPendingExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "camPendingExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool pendingimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("camtool_find_pendingimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereSQL(config->where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipProcessedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString cam_id = psMetadataLookupStr(&status, config->args, "-cam_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -cam_id"); + return false; + } + if (!cam_id) { + psError(PS_ERR_UNKNOWN, true, "-cam_id is required"); + return false; + } + + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + + psF32 sigma_ra = psMetadataLookupF32(&status, config->args, "-sigma_ra"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -sigma_ra"); + return false; + } + //if (isnan(sigma_ra)) { + // psError(PS_ERR_UNKNOWN, true, "-sigma_ra is required"); + // return false; + //} + + psF32 sigma_dec = psMetadataLookupF32(&status, config->args, "-sigma_dec"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -sigma_dec"); + return false; + } + //if (isnan(sigma_dec)) { + // psError(PS_ERR_UNKNOWN, true, "-sigma_dec is required"); + // return false; + //} + + psF32 zp_mean = psMetadataLookupF32(&status, config->args, "-zp_mean"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -zp_mean"); + return false; + } + //if (isnan(zp_mean)) { + // psError(PS_ERR_UNKNOWN, true, "-zp_mean is required"); + // return false; + //} + + psF32 zp_stdev = psMetadataLookupF32(&status, config->args, "-zp_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -zp_stdev"); + return false; + } + //if (isnan(zp_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-zp_stdev is required"); + // return false; + //} + + psF32 fwhm = psMetadataLookupF32(&status, config->args, "-fwhm"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fwhm"); + return false; + } + //if (isnan(fwhm)) { + // psError(PS_ERR_UNKNOWN, true, "-fwhm is required"); + // return false; + //} + + psF32 fwhm_range = psMetadataLookupF32(&status, config->args, "-fwhm_range"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fwhm_range"); + return false; + } + //if (isnan(fwhm_range)) { + // psError(PS_ERR_UNKNOWN, true, "-fwhm_range is required"); + // return false; + //} + + psS32 n_stars = psMetadataLookupS32(&status, config->args, "-n_stars"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -n_stars"); + return false; + } + if (n_stars < 0) { + psError(PS_ERR_UNKNOWN, true, "-n_stars is required"); + return false; + } + + psS32 n_extended = psMetadataLookupS32(&status, config->args, "-n_extended"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -n_extended"); + return false; + } + if (n_extended < 0) { + psError(PS_ERR_UNKNOWN, true, "-n_extended is required"); + return false; + } + + psS32 n_cr = psMetadataLookupS32(&status, config->args, "-n_cr"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -n_cr"); + return false; + } + if (n_cr < 0) { + psError(PS_ERR_UNKNOWN, true, "-n_cr is required"); + return false; + } + + psS32 n_astrom = psMetadataLookupS32(&status, config->args, "-n_astrom"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -n_astrom"); + return false; + } + if (n_astrom < 0) { + psError(PS_ERR_UNKNOWN, true, "-n_astrom is required"); + return false; + } + + // optional + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + // default + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + psString query = pxDataGet("camtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query, cam_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + camRunRow *pendingRow = camRunObjectFromMetadata(output->data[0]); + psFree(output); + // create a new camProcessedImfile object + camProcessedExpRow *row = camProcessedExpRowAlloc( + pendingRow->cam_id, + pendingRow->chip_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + sigma_ra, + sigma_dec, + zp_mean, + zp_stdev, + fwhm, + fwhm_range, + n_stars, + n_extended, + n_cr, + n_astrom, + path_base, + code + ); + psFree(pendingRow); + + // insert the new row into the camProcessedImfile table + if (!camProcessedExpInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + return false; + } + + // since there is only one exp per 'run' set camRun.state = 'stop' + if (!pxcamRunSetState(config, row->cam_id, "stop")) { + psError(PS_ERR_UNKNOWN, false, "failed to change camRun.state for cam_id: %" PRId64, row->cam_id); + psFree(row); + return false; + } + + psFree(row); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = pxDataGet("camtool_find_processedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND camProcessedExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND camProcessedExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "camProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool updateprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + if (code == INT16_MAX) { + psError(PS_ERR_UNKNOWN, true, "-code is required"); + return false; + } + + if (!pxSetFaultCode(config->dbh, "camProcessedExp", config->where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + return false; + } + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString label = psMetadataLookupStr(&status, config->args, "-label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label"); + return false; + } + if (!label) { + psError(PS_ERR_UNKNOWN, true, "-label is required"); + return false; + } + + if (!camMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = psStringCopy("SELECT * FROM camMask"); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("camtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "camMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString label = psMetadataLookupStr(&status, config->args, "-label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label"); + return false; + } + if (!label) { + psError(PS_ERR_UNKNOWN, true, "-label is required"); + return false; + } + + char *query = "DELETE FROM camMask WHERE label = '%s'"; + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + return true; +} Index: /branches/jh_branches/backtrack/ippTools/src/camtool.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/camtool.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/camtool.h (revision 22250) @@ -0,0 +1,40 @@ +/* + * camtool.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef CAMTOOL_H +#define CAMTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + CAMTOOL_MODE_NONE = 0x0, + CAMTOOL_MODE_QUEUE, + CAMTOOL_MODE_PENDINGEXP, + CAMTOOL_MODE_PENDINGIMFILE, + CAMTOOL_MODE_ADDPROCESSEDEXP, + CAMTOOL_MODE_PROCESSEDEXP, + CAMTOOL_MODE_UPDATEPROCESSEDEXP, + CAMTOOL_MODE_BLOCK, + CAMTOOL_MODE_MASKED, + CAMTOOL_MODE_UNBLOCK +} camtoolMode; + +pxConfig *camtoolConfig(pxConfig *config, int argc, char **argv); + +#endif // CAMTOOL_H Index: /branches/jh_branches/backtrack/ippTools/src/camtoolConfig.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/camtoolConfig.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/camtoolConfig.c (revision 22250) @@ -0,0 +1,320 @@ +/* + * camtoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include + +#include "pxtools.h" +#include "camtool.h" + +pxConfig *camtoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -queue + psMetadata *queueArgs = psMetadataAlloc(); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-chip_id", 0, + "search by chip_id", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search by exp_tag", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_id", 0, + "search by exp_id", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-inst", 0, + "search by camera", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-telescope", 0, + "search by telescope", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-dateobs", 0, + "search by observation time", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_type", 0, + "search by exposure type", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-imfiles", 0, + "search by number of imfiles", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-workdir", 0, + "search by workdir", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-filter", 0, + "search by filter ", NULL); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-airmass", 0, + "search by airmass", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-ra", 0, + "search by RA", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-decl", 0, + "search by DEC", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-exp_time", 0, + "search by exposure time", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg", 0, + "search by exposure background", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "search by exposure background stdev", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "search by exposure background mean stdev", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-alt", 0, + "search by altitute", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-az", 0, + "search by azimuth", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-ccd_temp", 0, + "search by ccd tempature", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-posang", 0, + "search by rotator position angle", NAN); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-object", 0, + "search by exposure object", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_workdir", 0, + "define workdir", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_label", 0, + "define label", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_reduction", 0, + "define reduction class", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_expgroup", 0, + "define exposure group", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_dvodb", 0, + "define DVO db", NULL); + + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, + "search by camtool ID", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-chip_id", 0, + "search by chiptool ID", NULL); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + + // -pendingimfile + psMetadata *pendingimfileArgs = psMetadataAlloc(); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-cam_id", 0, + "search by camtool ID", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-chip_id", 0, + "search by chiptool ID", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class", 0, + "search by class", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search by class ID", NULL); + psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + + // -addprocessedexp + psMetadata *addprocessedexpArgs = psMetadataAlloc(); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, + "define camtool ID (required)", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-uri", 0, + "define URI (required)", NULL); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-sigma_ra", 0, + "define exposure E ra (required)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-sigma_dec", 0, + "define exposure E dec (required)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-zp_mean", 0, + "define zero point mean (required)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-zp_stdev", 0, + "define zero point stdev (required)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fwhm", 0, + "define FWHM in arcsec (required)", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-fwhm_range", 0, + "define FWHM range in arcsec (required)", NAN); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_stars", 0, + "define number of stars (required)", -1); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_extended", 0, + "define number of extended objects (required)", -1); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_cr", 0, + "define number of cosmic rays (required)", -1); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_astrom", 0, + "define number of astrometry reference objects (required)", -1); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, + "define banana 1", NULL); + psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + psMetadataAddBool(addprocessedexpArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + + // -processedexp + psMetadata *processedexpArgs = psMetadataAlloc(); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-cam_id", 0, + "search by camtool ID", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-chip_id", 0, + "search by chiptool ID", NULL); + psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + + // -updateprocessedexp + psMetadata *updateprocessedexpArgs = psMetadataAlloc(); + psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, + "search by camtool ID", NULL); + psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-chip_id", 0, + "search by chiptool ID", NULL); + psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-class", 0, + "search by class", NULL); + psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-class_id", 0, + "search by class ID", NULL); + psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-code", 0, + "set fault code (required)", INT16_MAX); + + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, + "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, + "name of a label to unmask (required)", NULL); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXTOOL_ADD_MODE("-queue", "", CAMTOOL_MODE_QUEUE, queueArgs); + PXTOOL_ADD_MODE("-pendingexp", "", CAMTOOL_MODE_PENDINGEXP, pendingexpArgs); + PXTOOL_ADD_MODE("-pendingimfile", "", CAMTOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + PXTOOL_ADD_MODE("-addprocessedexp", "", CAMTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs); + PXTOOL_ADD_MODE("-processedexp", "", CAMTOOL_MODE_PROCESSEDEXP, processedexpArgs); + PXTOOL_ADD_MODE("-updateprocessedexp", "", CAMTOOL_MODE_UPDATEPROCESSEDEXP,updateprocessedexpArgs); + PXTOOL_ADD_MODE("-block", "", CAMTOOL_MODE_BLOCK, blockArgs); + PXTOOL_ADD_MODE("-masked", "", CAMTOOL_MODE_MASKED, maskedArgs); + PXTOOL_ADD_MODE("-unblock", "", CAMTOOL_MODE_UNBLOCK, unblockArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // setup search criterion +#define addWhereStr(name) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, #name, 0, "==", str)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + + // generate SQL where clause + config->where = psMetadataAlloc(); + +{ + psString str = NULL; + bool status = false; + if ((str = psMetadataLookupStr(&status, config->args, "-cam_id"))) { + if (!psMetadataAddS64(config->where, PS_LIST_TAIL, "cam_id", 0, "==", (psS64)atoll(str))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item cam_id"); + psFree(config); + return NULL; + } + } +} + +{ + psString str = NULL; + bool status = false; + if ((str = psMetadataLookupStr(&status, config->args, "-chip_id"))) { + if (!psMetadataAddS64(config->where, PS_LIST_TAIL, "chip_id", 0, "==", (psS64)atoll(str))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item chip_id"); + psFree(config); + return NULL; + } + } +} + + // convert '-inst' to 'camera' + { + psString str = NULL; + bool status = false; + if ((str = psMetadataLookupStr(&status, config->args, "-inst"))) { + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, "camera", 0, "==", str)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); + psFree(config); + return NULL; + } + } + } + addWhereStr(telescope); + addWhereStr(exp_type); + { + int imfiles = 0; + bool status = false; + if ((imfiles = psMetadataLookupS32(&status, config->args, "-imfiles"))) { + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, "imfiles", 0, "==", imfiles)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item imfiles"); + psFree(config); + return NULL; + } + } + } + addWhereStr(class); + addWhereStr(class_id); + addWhereStr(filter); + + if (psListLength(config->where->list) < 1) { + psFree(config->where); + config->where = NULL; + } + + + // define Database handle, if used + config->dbh = pmConfigDB(config->modules); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/jh_branches/backtrack/ippTools/src/chiptool.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/chiptool.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/chiptool.c (revision 22250) @@ -0,0 +1,869 @@ +/* + * chiptool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "pxdata.h" + +#include "chiptool.h" +#include "camtool.h" + +static bool queueMode(pxConfig *config); +static bool pendingimfileMode(pxConfig *config); +static bool addprocessedimfileMode(pxConfig *config); +static bool processedimfileMode(pxConfig *config); +static bool updateprocessedimfileMode(pxConfig *config); +static bool blockMode(pxConfig *config); +static bool maskedMode(pxConfig *config); +static bool unblockMode(pxConfig *config); + +static bool chipProcessedCompleteExp(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) { + psLibInit(NULL); + + pxConfig *config = chiptoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(CHIPTOOL_MODE_QUEUE, queueMode); + MODECASE(CHIPTOOL_MODE_PENDINGIMFILE, pendingimfileMode); + MODECASE(CHIPTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileMode); + MODECASE(CHIPTOOL_MODE_PROCESSEDIMFILE, processedimfileMode); + MODECASE(CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE,updateprocessedimfileMode); + MODECASE(CHIPTOOL_MODE_BLOCK, blockMode); + MODECASE(CHIPTOOL_MODE_MASKED, maskedMode); + MODECASE(CHIPTOOL_MODE_UNBLOCK, unblockMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +#define ADDRENAMEPARAMSTR(from, to, oldname, newname) \ +{ \ + bool status = false; \ + psString str = psMetadataLookupStr(&status, from, "-" oldname); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -" oldname); \ + return false; \ + } \ + if (str) { \ + if (!psMetadataAddStr(to, PS_LIST_TAIL, newname, 0, "==", str)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " newname); \ + psFree(to); \ + return false; \ + } \ + } \ +} + +#define ADDPARAMSTR(from, to, name) ADDRENAMEPARAMSTR(from, to, name, name) + +#define ADDPARAMF(from, to, type, name) \ +{ \ + bool status = false; \ + ps##type var = psMetadataLookup##type(&status, from, "-" name); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -" name); \ + return false; \ + } \ + if (!isnan(var)) { \ + if (!psMetadataAdd##type(to, PS_LIST_TAIL, #name, 0, "==", var)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " name); \ + psFree(to); \ + return false; \ + } \ + } \ +} + +static bool queueMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + ADDPARAMSTR(config->args, where, "exp_tag"); + ADDPARAMSTR(config->args, where, "exp_id"); + ADDRENAMEPARAMSTR(config->args, where, "inst", "camera"); + ADDPARAMSTR(config->args, where, "telescope"); + ADDPARAMSTR(config->args, where, "dateobs"); + ADDPARAMSTR(config->args, where, "exp_type"); + ADDPARAMSTR(config->args, where, "imfiles"); + ADDPARAMSTR(config->args, where, "workdir"); + ADDPARAMSTR(config->args, where, "filter"); + ADDPARAMF(config->args, where, F32, "airmass"); + ADDPARAMF(config->args, where, F32, "ra"); + ADDPARAMF(config->args, where, F32, "decl"); + ADDPARAMF(config->args, where, F32, "exp_time"); + ADDPARAMF(config->args, where, F32, "bg"); + ADDPARAMF(config->args, where, F32, "bg_stdev"); + ADDPARAMF(config->args, where, F32, "bg_mean_stdev"); + ADDPARAMF(config->args, where, F64, "alt"); + ADDPARAMF(config->args, where, F64, "az"); + ADDPARAMF(config->args, where, F32, "ccd_temp"); + ADDPARAMF(config->args, where, F64, "posang"); + ADDPARAMSTR(config->args, where, "object"); + + if (where->list->n < 1) { + psFree(where); + where = NULL; + } + + bool status = false; + psString workdir = psMetadataLookupStr(&status, config->args, "-set_workdir"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_workdir"); + return false; + } + + psString label = psMetadataLookupStr(&status, config->args, "-set_label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_label"); + return false; + } + + psString reduction = psMetadataLookupStr(&status, config->args, "-set_reduction"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_reduction"); + return false; + } + + psString expgroup = psMetadataLookupStr(&status, config->args, "-set_expgroup"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_expgroup"); + return false; + } + + psString dvodb = psMetadataLookupStr(&status, config->args, "-set_dvodb"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_dvodb"); + return false; + } + + // find the exp_tag of all the exposures that we want to queue up. + psString query = pxDataGet("chiptool_find_rawexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); + psFree(where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // loop over our list of exp_tags + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + psString exp_tag = psMetadataLookupStr(&status, md, "exp_tag"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_tag"); + psFree(output); + return false; + } + + // queue the exp + if (!pxchipQueueByExpTag(config, exp_tag, workdir, label, reduction, expgroup, dvodb)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue exp_tag: %s", exp_tag); + psFree(output); + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool pendingimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + // XXX does this need to be constrained so that it won't return any results + // if a match chipPendingExp hasn't been registered? + psString query = pxDataGet("chiptool_pendingimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "chipInputImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipInputImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + // chip_id, ext_tag, class_id are required + psString chip_id = psMetadataLookupStr(&status, config->args, "-chip_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -chip_id"); + return false; + } + if (!chip_id) { + psError(PS_ERR_UNKNOWN, true, "-chip_id is required"); + return false; + } + + psString exp_tag = psMetadataLookupStr(&status, config->args, "-exp_tag"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_tag"); + return false; + } + if (!exp_tag) { + psError(PS_ERR_UNKNOWN, true, "-exp_tag is required"); + return false; + } + + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + return false; + } + if (!class_id) { + psError(PS_ERR_UNKNOWN, true, "-class_id is required"); + return false; + } + + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + + psF32 bg = psMetadataLookupF32(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + + psF32 bg_stdev = psMetadataLookupF32(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + + psF32 bg_mean_stdev = psMetadataLookupF32(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + + psF32 bias = psMetadataLookupF32(&status, config->args, "-bias"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bias"); + return false; + } + + psF32 bias_stdev = psMetadataLookupF32(&status, config->args, "-bias_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bias_stdev"); + return false; + } + + psF32 fringe_0 = psMetadataLookupF32(&status, config->args, "-fringe_0"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_0"); + return false; + } + + psF32 fringe_1 = psMetadataLookupF32(&status, config->args, "-fringe_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_1"); + return false; + } + + psF32 fringe_2 = psMetadataLookupF32(&status, config->args, "-fringe_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_2"); + return false; + } + + psF32 sigma_ra = psMetadataLookupF32(&status, config->args, "-sigma_ra"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -sigma_ra"); + return false; + } + + psF32 sigma_dec = psMetadataLookupF32(&status, config->args, "-sigma_dec"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -sigma_dec"); + return false; + } + + psF32 ap_resid = psMetadataLookupF32(&status, config->args, "-ap_resid"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ap_resid"); + return false; + } + + psF32 ap_resid_stdev = psMetadataLookupF32(&status, config->args, "-ap_resid_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ap_resid_stdev"); + return false; + } + + psF32 fwhm = psMetadataLookupF32(&status, config->args, "-fwhm"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fwhm"); + return false; + } + + psF32 fwhm_range = psMetadataLookupF32(&status, config->args, "-fwhm_range"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fwhm_range"); + return false; + } + + psS32 n_stars = psMetadataLookupS32(&status, config->args, "-n_stars"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -n_stars"); + return false; + } + + psS32 n_extended = psMetadataLookupS32(&status, config->args, "-n_extended"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -n_extended"); + return false; + } + + psS32 n_cr = psMetadataLookupS32(&status, config->args, "-n_cr"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -n_cr"); + return false; + } + + psS32 n_astrom = psMetadataLookupS32(&status, config->args, "-n_astrom"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -n_astrom"); + return false; + } + + // optional + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + chipProcessedImfileRow *imfile = chipProcessedImfileRowAlloc( + (psU64)atoll(chip_id), + exp_tag, + class_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + bias, + bias_stdev, + fringe_0, + fringe_1, + fringe_2, + sigma_ra, + sigma_dec, + ap_resid, + ap_resid_stdev, + fwhm, + fwhm_range, + n_stars, + n_extended, + n_cr, + n_astrom, + path_base, + code + ); + if (!imfile) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to alloc chipProcessedImfileRow object"); + return false; + } + if (!chipProcessedImfileInsertObject(config->dbh, imfile)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(imfile); + return false; + } + + + + // XXX I've decided to make the transaction cover the Exp migration as + // well. Otherwise, if the last imfile in an exp is moved and the exp + // migration fails then the data base is left in a satuation where the exp + // migration can't happen. + + if (!chipProcessedCompleteExp(config)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + // XXX does this need to be constrained so that it won't return any results + // if a match chipPendingExp hasn't been registered? + psString query = pxDataGet("chiptool_processedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "chipProcessedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND chipProcessedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipProcessedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool updateprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + if (code == INT16_MAX) { + psError(PS_ERR_UNKNOWN, true, "-code is required"); + return false; + } + + if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", config->where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + return false; + } + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString label = psMetadataLookupStr(&status, config->args, "-label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label"); + return false; + } + if (!label) { + psError(PS_ERR_UNKNOWN, true, "-label is required"); + return false; + } + + if (!chipMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = psStringCopy("SELECT * FROM chipMask"); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString label = psMetadataLookupStr(&status, config->args, "-label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label"); + return false; + } + if (!label) { + psError(PS_ERR_UNKNOWN, true, "-label is required"); + return false; + } + + char *query = "DELETE FROM chipMask WHERE label = '%s'"; + + if (!p_psDBRunQuery(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + return true; +} + + +static bool chipProcessedCompleteExp(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // look for completed chipPendingExp + // migrate them to chipProccessedExp & camPendingExp + psString query = pxDataGet("chiptool_completely_processed_exp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + chipRunRow *chipRun = chipRunObjectFromMetadata(row); + // set chipRun.state to 'stop' + if (!pxchipRunSetState(config, chipRun->chip_id, "stop")) { + psError(PS_ERR_UNKNOWN, false, "failed to change chipRun.state for chip_id: %" PRId64, chipRun->chip_id); + psFree(chipRun); + psFree(output); + return false; + } + + // camQueueChipID() can only be run after chipRun.state has been set to + // stop + if (!pxcamQueueByChipID(config, + chipRun->chip_id, + chipRun->workdir, + chipRun->label, + chipRun->reduction, + chipRun->expgroup, + chipRun->dvodb + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to queue camPendingExp"); + psFree(chipRun); + psFree(output); + return false; + } + psFree(chipRun); + } + + psFree(output); + + return true; +} Index: /branches/jh_branches/backtrack/ippTools/src/chiptool.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/chiptool.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/chiptool.h (revision 22250) @@ -0,0 +1,40 @@ +/* + * chiptool.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef CHIPTOOL_H +#define CHIPTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + CHIPTOOL_MODE_NONE = 0x0, + CHIPTOOL_MODE_QUEUE, + CHIPTOOL_MODE_PENDINGIMFILE, + CHIPTOOL_MODE_ADDPROCESSEDIMFILE, + CHIPTOOL_MODE_PROCESSEDIMFILE, + CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE, + CHIPTOOL_MODE_BLOCK, + CHIPTOOL_MODE_MASKED, + CHIPTOOL_MODE_UNBLOCK, + CHIPTOOL_MODE_RETRYPROCESSEDIMFILE +} chiptoolMode; + +pxConfig *chiptoolConfig(pxConfig *config, int argc, char **argv); + +#endif // CHIPTOOL_H Index: /branches/jh_branches/backtrack/ippTools/src/chiptoolConfig.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/chiptoolConfig.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/chiptoolConfig.c (revision 22250) @@ -0,0 +1,337 @@ +/* + * chiptoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "chiptool.h" + +pxConfig *chiptoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -queue + psMetadata *queueArgs = psMetadataAlloc(); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search by exp_tag", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_id", 0, + "search by exp_id", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-inst", 0, + "search by camera", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-telescope", 0, + "search by telescope", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-dateobs", 0, + "search by observation time", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_type", 0, + "search by exposure type", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-imfiles", 0, + "search by number of imfiles", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-workdir", 0, + "search by workdir", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-filter", 0, + "search by filter ", NULL); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-airmass", 0, + "search by airmass", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-ra", 0, + "search by RA", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-decl", 0, + "search by DEC", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-exp_time", 0, + "search by exposure time", NAN); + + // XXX EAM : are these selecting based on the rawExp value or the chipImfile value? + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg", 0, + "search by exposure background", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "search by exposure background stdev", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "search by exposure background mean stdev", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-alt", 0, + "search by altitute", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-az", 0, + "search by azimuth", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-ccd_temp", 0, + "search by ccd tempature", NAN); + psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-posang", 0, + "search by rotator position angle", NAN); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-object", 0, + "search by exposure object", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_workdir", 0, + "define workdir", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_label", 0, + "define label", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_reduction", 0, + "define reduction class", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_expgroup", 0, + "define exposure group", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_dvodb", 0, + "define DVO db", NULL); + + + // -pendingimfile + psMetadata *pendingimfileArgs = psMetadataAlloc(); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-chip_id", 0, + "search by chip ID", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search by exposure ID", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class", 0, + "search by class", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search by class ID", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-inst", 0, + "search by camera of interest", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-filter", 0, + "search by filter of interest", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-uri", 0, + "search by URL", NULL); + psMetadataAddU64(pendingimfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addprocessedimfile + psMetadata *addprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, + "define chip ID (required)", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-exp_tag", 0, + "define exposure ID", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "define class ID (requied)", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-uri", 0, + "define URL", NULL); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-bias", 0, + "define bias", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-bias_stdev", 0, + "define bias stdev", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_0", 0, + "define fringe term 0", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_1", 0, + "define fringe term 1", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_2", 0, + "define fringe term 2", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-sigma_ra", 0, + "define error in RA", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-sigma_dec", 0, + "define error in DEC", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-ap_resid", 0, + "define aperture residual", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-ap_resid_stdev", 0, + "define aperture residual stdev", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fwhm", 0, + "define fwhm (arcsec)", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-fwhm_range", 0, + "define fwhm range (min to max)", NAN); + + psMetadataAddS32(addprocessedimfileArgs, PS_LIST_TAIL, "-n_stars", 0, + "define number of stars in image", 0); + psMetadataAddS32(addprocessedimfileArgs, PS_LIST_TAIL, "-n_extended", 0, + "define number of extended objects in image", 0); + psMetadataAddS32(addprocessedimfileArgs, PS_LIST_TAIL, "-n_cr", 0, + "define number of cosmic rays in image", 0); + psMetadataAddS32(addprocessedimfileArgs, PS_LIST_TAIL, "-n_astrom", 0, + "define number of astrometry stars used", 0); + + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + + // -processedimfile + psMetadata *processedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, + "define chip ID", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-exp_tag", 0, + "define exposure ID", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-class", 0, + "define class", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "define class ID", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-inst", 0, + "define camera of interest", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-filter", 0, + "define filter of interest", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-uri", 0, + "define URL", NULL); + psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -updateprocessedimfile + psMetadata *updateprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, + "define chip ID", NULL); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search by exposure ID", NULL); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class", 0, + "search by class", NULL); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search by class ID", NULL); + psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, + "set fault code (required)", INT16_MAX); + + // -retryprocessedimfile + psMetadata *retryprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(retryprocessedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, + "define chip ID (required)", NULL); + psMetadataAddStr(retryprocessedimfileArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search by exposure ID", NULL); + psMetadataAddStr(retryprocessedimfileArgs, PS_LIST_TAIL, "-class", 0, + "search by class", NULL); + psMetadataAddStr(retryprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search by class ID", NULL); + + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, + "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, + "name of a label to unmask (required)", NULL); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXTOOL_ADD_MODE("-queue", "", CHIPTOOL_MODE_QUEUE, queueArgs); + PXTOOL_ADD_MODE("-pendingimfile", "", CHIPTOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + PXTOOL_ADD_MODE("-addprocessedimfile", "", CHIPTOOL_MODE_ADDPROCESSEDIMFILE,addprocessedimfileArgs); + PXTOOL_ADD_MODE("-processedimfile", "", CHIPTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); + PXTOOL_ADD_MODE("-updateprocessedimfile", "", CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileArgs); + PXTOOL_ADD_MODE("-retryprocessedimfile", "", CHIPTOOL_MODE_RETRYPROCESSEDIMFILE, retryprocessedimfileArgs); + PXTOOL_ADD_MODE("-block", "", CHIPTOOL_MODE_BLOCK, blockArgs); + PXTOOL_ADD_MODE("-masked", "", CHIPTOOL_MODE_MASKED, maskedArgs); + PXTOOL_ADD_MODE("-unblock", "", CHIPTOOL_MODE_UNBLOCK, unblockArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // setup search criterion +#define addWhereStr(name) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, #name, 0, "==", str)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + + + + // generate SQL where clause + config->where = psMetadataAlloc(); + +{ + psString str = NULL; + bool status = false; + if ((str = psMetadataLookupStr(&status, config->args, "-chip_id"))) { + if (!psMetadataAddS64(config->where, PS_LIST_TAIL, "chip_id", 0, "==", (psS64)atoll(str))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item chip_id"); + psFree(config); + return NULL; + } + } +} + + addWhereStr(exp_tag); + // convert '-inst' to 'camera' + { + psString str = NULL; + bool status = false; + if ((str = psMetadataLookupStr(&status, config->args, "-inst"))) { + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, "camera", 0, "==", str)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); + psFree(config); + return NULL; + } + } + } + + + addWhereStr(filter); + addWhereStr(class); + addWhereStr(class_id); + + /* + // psMetadataConfig does not support times yet + if (!psTimeIsZero(config->start) && !psTimeIsZero(config->stop)) { + psMetadataAddTime (where, PS_LIST_TAIL, "TIME_START", 0, "<", config->stop); + psMetadataAddTime (where, PS_LIST_TAIL, "TIME_STOP", 0, ">", config->start); + } + */ + + if (config->where->list->n < 1) { + psFree(config->where); + config->where = NULL; + } + + // define Database handle, if used + config->dbh = pmConfigDB(config->modules); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/jh_branches/backtrack/ippTools/src/detselect.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/detselect.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/detselect.c (revision 22250) @@ -0,0 +1,293 @@ +/* + * detselect.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "detselect.h" + +static bool searchMode(pxConfig *config); +static bool selectMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = detselectConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(DETSELECT_MODE_SEARCH, searchMode); + MODECASE(DETSELECT_MODE_SELECT, selectMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool searchMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + // airmass_min < airmass < airmass_max + { + bool status = false; + psF32 airmass = psMetadataLookupF32(&status, config->args, "-airmass"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass"); + psFree(where); + return false; + } + + if (!isnan(airmass)) { + psMetadataAddF32(where, PS_LIST_TAIL, "airmass_min", 0, "<=", airmass); + psMetadataAddF32(where, PS_LIST_TAIL, "airmass_max", 0, ">=", airmass); + } + } + + // exp_time_min < exp_time < exp_time_max + { + bool status = false; + psF32 exp_time = psMetadataLookupF32(&status, config->args, "-exp_time"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time"); + psFree(where); + return false; + } + + if (!isnan(exp_time)) { + psMetadataAddF32(where, PS_LIST_TAIL, "exp_time_min", 0, "<=", exp_time); + psMetadataAddF32(where, PS_LIST_TAIL, "exp_time_max", 0, ">=", exp_time); + } + } + + // ccd_temp_min < ccd_temp < ccd_temp_max + { + bool status = false; + psF32 ccd_temp = psMetadataLookupF32(&status, config->args, "-ccd_temp"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp"); + psFree(where); + return false; + } + + if (!isnan(ccd_temp)) { + psMetadataAddF32(where, PS_LIST_TAIL, "ccd_temp_min", 0, "<=", ccd_temp); + psMetadataAddF32(where, PS_LIST_TAIL, "ccd_temp_max", 0, ">=", ccd_temp); + } + } + + { + bool status = false; + psF64 posang = psMetadataLookupF64(&status, config->args, "-posang"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang"); + psFree(where); + return false; + } + + if (!isnan(posang)) { + psMetadataAddF32(where, PS_LIST_TAIL, "posang_min", 0, "<=", posang); + psMetadataAddF32(where, PS_LIST_TAIL, "posang_max", 0, ">=", posang); + } + } + + // time_begin < time < time_end + { + bool status = false; + psString timeStr = psMetadataLookupStr(&status, config->args, "-time"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time"); + psFree(where); + return false; + } + + if (timeStr) { + psTime *time = psTimeFromISO(timeStr, PS_TIME_UTC); + if (!time) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", timeStr); + psFree(where); + return false; + } + + // the == NULL tests invokes some psDB magic to make an OR + // conditional query + psMetadataAddTime(where, PS_LIST_TAIL, "time_begin", 0, "<=", time); + psMetadataAddTime(where, PS_LIST_TAIL, "time_begin", PS_META_DUPLICATE_OK, "==", NULL); + psMetadataAddTime(where, PS_LIST_TAIL, "time_end", 0, ">=", time); + psMetadataAddTime(where, PS_LIST_TAIL, "time_end", PS_META_DUPLICATE_OK, "==", NULL); + psFree(time); + } + } + + psString query = pxDataGet("detselect_search.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detRun"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detRun"); + psFree(where); + if (whereClause) { + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + } + + // XXX this needs to be more controlled: we should specifically choose the + // single detrend image which matches all criteria and has the latest + // insertion date + psStringAppend(&query, " ORDER BY detRun.registered DESC LIMIT 1"); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no detrend exposures found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool selectMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("detselect_select.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detNormalizedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no detNormalizedImfile rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detNormalizedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} Index: /branches/jh_branches/backtrack/ippTools/src/detselect.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/detselect.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/detselect.h (revision 22250) @@ -0,0 +1,33 @@ +/* + * detselect.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef DETSELECT_H +#define DETSELECT_H 1 + +#include "pxtools.h" + +typedef enum { + DETSELECT_MODE_NONE = 0x0, + DETSELECT_MODE_SEARCH, + DETSELECT_MODE_SELECT, +} detselectMode; + +pxConfig *detselectConfig(pxConfig *config, int argc, char **argv); + +#endif // DETSELECT_H Index: /branches/jh_branches/backtrack/ippTools/src/detselectConfig.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/detselectConfig.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/detselectConfig.c (revision 22250) @@ -0,0 +1,226 @@ +/* + * detselectConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "pxtools.h" +#include "detselect.h" + +pxConfig *detselectConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -search + psMetadata *searchArgs = psMetadataAlloc(); + psMetadataAddStr(searchArgs, PS_LIST_TAIL, "-inst", 0, + "search by camera", NULL); + psMetadataAddStr(searchArgs, PS_LIST_TAIL, "-telescope", 0, + "search by telescope", NULL); + psMetadataAddStr(searchArgs, PS_LIST_TAIL, "-det_type", 0, + "search by detrend type", NULL); + psMetadataAddStr(searchArgs, PS_LIST_TAIL, "-filter", 0, + "search by filter", NULL); + psMetadataAddF32(searchArgs, PS_LIST_TAIL, "-airmass", 0, + "define airmass", NAN); + psMetadataAddF32(searchArgs, PS_LIST_TAIL, "-exp_time", 0, + "search by exposure time", NAN); + psMetadataAddF32(searchArgs, PS_LIST_TAIL, "-ccd_temp", 0, + "search by ccd tempature", NAN); + psMetadataAddF64(searchArgs, PS_LIST_TAIL, "-posang", 0, + "search by rotator position angle", NAN); + psMetadataAddStr(searchArgs, PS_LIST_TAIL, "-time", 0, + "define time for desired detrend data", NULL); + psMetadataAddBool(searchArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -select + psMetadata *selectArgs = psMetadataAlloc(); + psMetadataAddStr(selectArgs, PS_LIST_TAIL, "-det_id", 0, + "search by detrend ID", NULL); + psMetadataAddS32(selectArgs, PS_LIST_TAIL, "-iteration", 0, + "search by iteration number", 0); + psMetadataAddStr(selectArgs, PS_LIST_TAIL, "-class_id", 0, + "search by class ID", NULL); + psMetadataAddBool(selectArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXTOOL_ADD_MODE("-search", "search for an appropriate detrend", DETSELECT_MODE_SEARCH, searchArgs); + PXTOOL_ADD_MODE("-select", "retreive detrend information", DETSELECT_MODE_SELECT, selectArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // setup search criterion +#define addWhereStr(name) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, #name, 0, "==", str)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define addWhereS32(name) \ +{ \ + psS32 s32 = 0; \ + bool status = false; \ + if ((s32= psMetadataLookupS32(&status, config->args, "-" #name))) { \ + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, #name, 0, "==", s32)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define addWhereF32(name) \ +{ \ + psF32 var = 0; \ + bool status = false; \ + if ((var = psMetadataLookupF32(&status, config->args, "-" #name))) { \ + if (!isnan(var)) { \ + if (!psMetadataAddF32(config->where, PS_LIST_TAIL, #name, 0, "==", var)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ + } \ +} + +#define addWhereF64(name) \ +{ \ + psF64 var = 0; \ + bool status = false; \ + if ((var = psMetadataLookupF64(&status, config->args, "-" #name))) { \ + if (!isnan(var)) { \ + if (!psMetadataAddF64(config->where, PS_LIST_TAIL, #name, 0, "==", var)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ + } \ +} + +#define addWhereTimeStr(name) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + psTime *time = psTimeFromISO(str, PS_TIME_UTC); \ + if (!time) { \ + psError(PS_ERR_UNKNOWN, false, "failed to convert " #name " into a psTime object"); \ + psFree(config); \ + return NULL; \ + } \ + psMetadataItem *item = psMetadataLookup(config->args, "-" #name); \ + if (item) { \ + str = item->comment; \ + } else { \ + str = NULL; \ + } \ + if (!psMetadataAddTime(config->where, PS_LIST_TAIL, #name, 0, str, time)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + psFree(time); \ + } \ +} + + // generate SQL where clause + config->where = psMetadataAlloc(); + + addWhereStr(det_id); + addWhereS32(iteration); + addWhereStr(exp_tag); + // convert '-inst' to 'camera' + { + psString str = NULL; + bool status = false; + if ((str = psMetadataLookupStr(&status, config->args, "-inst"))) { + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, "camera", 0, "==", str)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); + psFree(config); + return NULL; + } + } + } + addWhereStr(telescope); + addWhereStr(det_type); + addWhereStr(exp_type); + addWhereS32(imfiles); + addWhereStr(class); + addWhereStr(class_id); + addWhereStr(filter); +// addWhereF32(exp_time); +// addWhereF64(ccd_temp); +// addWhereF32(airmass); +// addWhereTimeStr(use_begin); +// addWhereTimeStr(use_end); + + if (config->where->list->n < 1) { + psFree(config->where); + config->where = NULL; + } + + + // define Database handle, if used + config->dbh = pmConfigDB(config->modules); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/jh_branches/backtrack/ippTools/src/dettool.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/dettool.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/dettool.c (revision 22250) @@ -0,0 +1,7111 @@ +/* + * dettool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include + +#include "pxtools.h" +#include "dettool.h" + +static bool pendingMode(pxConfig *config); +static bool definebytagMode(pxConfig *config); +static bool definebyqueryMode(pxConfig *config); +static bool definebydetrunMode(pxConfig *config); +static bool runsMode(pxConfig *config); +static bool childlessrunMode(pxConfig *config); +static bool inputMode(pxConfig *config); +static bool rawMode(pxConfig *config); +static bool toprocessedimfileMode(pxConfig *config); +static bool addprocessedimfileMode(pxConfig *config); +static bool toprocessedexpMode(pxConfig *config); +static bool addprocessedexpMode(pxConfig *config); +static bool processedexpMode(pxConfig *config); +static bool tostackedMode(pxConfig *config); +static bool processedimfileMode(pxConfig *config); +static bool addstackedMode(pxConfig *config); +static bool stackedMode(pxConfig *config); +static bool tonormalizeMode(pxConfig *config); +static bool addnormalizedstatMode(pxConfig *config); +static bool tonormalizedstatMode(pxConfig *config); +static bool addnormalizedimfileMode(pxConfig *config); +static bool tonormalizedexpMode(pxConfig *config); +static bool addnormalizedexpMode(pxConfig *config); +static bool normalizedexpMode(pxConfig *config); +static bool toresidimfileMode(pxConfig *config); +static bool addresidimfileMode(pxConfig *config); +static bool normalizedimfileMode(pxConfig *config); +static bool toresidexpMode(pxConfig *config); +static bool residimfileMode(pxConfig *config); +static bool addresidexpMode(pxConfig *config); +static bool residexpMode(pxConfig *config); +static bool residdetrunMode(pxConfig *config); +static bool updateresidexpMode(pxConfig *config); +static bool adddetrunsummaryMode(pxConfig *config); +static bool detrunsummaryMode(pxConfig *config); +static bool updatedetrunMode(pxConfig *config); +static bool rerunMode(pxConfig *config); +static bool register_detrendMode(pxConfig *config); +static bool register_detrend_imfileMode(pxConfig *config); + +static detNormalizedStatImfileRow *detStackedToDetNormalizedStatImfile(pxConfig *config, detStackedImfileRow *stackedImfile); +static detNormalizedImfileRow *detNormalizedStatToDetNormalizedmfile(pxConfig *config, detNormalizedStatImfileRow *statImfile); +static detResidImfileRow *detNormalizedToDetResidImfile(pxConfig *config, detNormalizedImfileRow *normalizedImfile); +static detResidExpRow *mdToDetResidExp(pxConfig *config, psMetadata *row); +static detRunSummaryRow *mdToDetRunSummary(pxConfig *config, psMetadata *row); +//static psArray *validDetInputClassIds(pxConfig *config, const char *det_id); +//static psArray *searchInputImfiles(pxConfig *config, const char *det_id); +static detInputExpRow *rawDetrenTodetInputExpRow(rawExpRow *rawExp, psS32 det_id, psS32 iteration); +static psArray *searchRawImfiles(pxConfig *config, psMetadata *where); +static psS32 incrementIteration(pxConfig *config, const char *det_id); +static bool setDetRunState(pxConfig *config, const char *det_id, const char *state); +static bool isValidMode(pxConfig *config, const char *mode); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = dettoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(DETTOOL_MODE_PENDING, pendingMode); + MODECASE(DETTOOL_MODE_DEFINEBYTAG, definebytagMode); + MODECASE(DETTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(DETTOOL_MODE_DEFINEBYDETRUN, definebydetrunMode); + MODECASE(DETTOOL_MODE_RUNS, runsMode); + MODECASE(DETTOOL_MODE_CHILDLESSRUN, childlessrunMode); + MODECASE(DETTOOL_MODE_INPUT, inputMode); + MODECASE(DETTOOL_MODE_RAW, rawMode); + MODECASE(DETTOOL_MODE_TOPROCESSEDIMFILE,toprocessedimfileMode); + MODECASE(DETTOOL_MODE_ADDPROCESSEDIMFILE,addprocessedimfileMode); + MODECASE(DETTOOL_MODE_PROCESSEDIMFILE, processedimfileMode); + MODECASE(DETTOOL_MODE_TOPROCESSEDEXP, toprocessedexpMode); + MODECASE(DETTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpMode); + MODECASE(DETTOOL_MODE_PROCESSEDEXP, processedexpMode); + MODECASE(DETTOOL_MODE_TOSTACKED, tostackedMode); + MODECASE(DETTOOL_MODE_ADDSTACKED, addstackedMode); + MODECASE(DETTOOL_MODE_STACKED, stackedMode); + MODECASE(DETTOOL_MODE_TONORMALIZE, tonormalizeMode); + MODECASE(DETTOOL_MODE_ADDNORMALIZEDSTAT,addnormalizedstatMode); + MODECASE(DETTOOL_MODE_TONORMALIZEDSTAT, tonormalizedstatMode); + MODECASE(DETTOOL_MODE_ADDNORMALIZEDIMFILE,addnormalizedimfileMode); + MODECASE(DETTOOL_MODE_NORMALIZEDIMFILE, normalizedimfileMode); + MODECASE(DETTOOL_MODE_TONORMALIZEDEXP, tonormalizedexpMode); + MODECASE(DETTOOL_MODE_ADDNORMALIZEDEXP, addnormalizedexpMode); + MODECASE(DETTOOL_MODE_NORMALIZEDEXP, normalizedexpMode); + MODECASE(DETTOOL_MODE_TORESIDIMFILE, toresidimfileMode); + MODECASE(DETTOOL_MODE_ADDRESIDIMFILE, addresidimfileMode); + MODECASE(DETTOOL_MODE_TORESIDEXP, toresidexpMode); + MODECASE(DETTOOL_MODE_RESIDIMFILE, residimfileMode); + MODECASE(DETTOOL_MODE_ADDRESIDEXP, addresidexpMode); + MODECASE(DETTOOL_MODE_RESIDEXP, residexpMode); + MODECASE(DETTOOL_MODE_RESIDDETRUN, residdetrunMode); + MODECASE(DETTOOL_MODE_UPDATERESIDEXP, updateresidexpMode); + MODECASE(DETTOOL_MODE_ADDDETRUNSUMMARY, adddetrunsummaryMode); + MODECASE(DETTOOL_MODE_DETRUNSUMMARY, detrunsummaryMode); + MODECASE(DETTOOL_MODE_UPDATEDETRUN, updatedetrunMode); + MODECASE(DETTOOL_MODE_RERUN, rerunMode); + MODECASE(DETTOOL_MODE_REGISTER_DETREND, register_detrendMode); + MODECASE(DETTOOL_MODE_REGISTER_DETREND_IMFILE, register_detrend_imfileMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool pendingMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psString query = psStringCopy( + "SELECT" + " rawExp.*" + " FROM rawExp" + " LEFT JOIN detInputExp" + " ON rawExp.exp_tag = detInputExp.exp_tag" + " WHERE" + " detInputExp.exp_tag IS NULL" + " AND rawExp.obstype != 'object'" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool definebytagMode(pxConfig *config) +{ + bool status = false; + + PS_ASSERT_PTR_NON_NULL(config, false); + + // what type of detRun is this? + // det_type is required + psString det_type = psMetadataLookupStr(&status, config->args, "-det_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_type"); + return false; + } + if (!det_type) { + psError(PS_ERR_UNKNOWN, true, "-det_type is required"); + return false; + } + + // optional + psString filelevel = psMetadataLookupStr(&status, config->args, "-filelevel"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filelevel"); + return false; + } + + psString workdir = psMetadataLookupStr(&status, config->args, "-workdir"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -workdir"); + return false; + } + if (!workdir) { + psError(PS_ERR_UNKNOWN, true, "-workdir is required"); + return false; + } + + // optional + psString mode = psMetadataLookupStr(&status, config->args, "-mode"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -mode"); + return false; + } + // check mode + if (mode && !isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalud mode"); + return false; + } + psString camera = psMetadataLookupStr(&status, config->args, "-inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst"); + return false; + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope"); + return false; + } + + psString exp_type = psMetadataLookupStr(&status, config->args, "-exp_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_type"); + return false; + } + + psString filter = psMetadataLookupStr(&status, config->args, "-filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter"); + return false; + } + + psF32 airmass_min = psMetadataLookupF32(&status, config->args, "-airmass_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass_min"); + return false; + } + + psF32 airmass_max = psMetadataLookupF32(&status, config->args, "-airmass_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass_max"); + return false; + } + + psF32 exp_time_min = psMetadataLookupF32(&status, config->args, "-exp_time_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time_min"); + return false; + } + + psF32 exp_time_max = psMetadataLookupF32(&status, config->args, "-exp_time_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time_max"); + return false; + } + + psF32 ccd_temp_min = psMetadataLookupF32(&status, config->args, "-ccd_temp_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp_min"); + return false; + } + + psF32 ccd_temp_max = psMetadataLookupF32(&status, config->args, "-ccd_temp_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp_max"); + return false; + } + + psF64 posang_min = psMetadataLookupF32(&status, config->args, "-posang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang_min"); + return false; + } + + psF64 posang_max = psMetadataLookupF32(&status, config->args, "-posang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang_max"); + return false; + } + psF64 solang_min = psMetadataLookupF32(&status, config->args, "-solang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -solang_min"); + return false; + } + + psF64 solang_max = psMetadataLookupF32(&status, config->args, "-solang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -solang_max"); + return false; + } + + psTime *registered = NULL; + { + psString registeredStr = psMetadataLookupStr(&status, config->args, "-registered"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -registered"); + return false; + } + // pass through NULL as this is an optional field + if (registeredStr) { + registered = psTimeFromISO(registeredStr, PS_TIME_UTC); + if (!registered) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", registeredStr); + return false; + } + } else { + registered = NULL; + } + } + + psTime *time_begin = NULL; + { + psString time_beginStr = psMetadataLookupStr(&status, config->args, "-time_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time_begin"); + return false; + } + // pass through NULL as this is an optional field + if (time_beginStr) { + time_begin = psTimeFromISO(time_beginStr, PS_TIME_UTC); + if (!time_begin) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", time_beginStr); + return false; + } + } else { + time_begin = NULL; + } + } + + psTime *time_end = NULL; + { + psString time_endStr = psMetadataLookupStr(&status, config->args, "-time_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time_end"); + return false; + } + // pass through NULL as this is an optional field + if (time_endStr) { + time_end = psTimeFromISO(time_endStr, PS_TIME_UTC); + if (!time_end) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", time_endStr); + return false; + } + } else { + time_end = NULL; + } + } + + psTime *use_begin = NULL; + { + psString use_beginStr = psMetadataLookupStr(&status, config->args, "-use_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -use_begin"); + return false; + } + // pass through NULL as this is an optional field + if (use_beginStr) { + use_begin = psTimeFromISO(use_beginStr, PS_TIME_UTC); + if (!use_begin) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", use_beginStr); + return false; + } + } else { + use_begin = NULL; + } + } + + psTime *use_end = NULL; + { + psString use_endStr = psMetadataLookupStr(&status, config->args, "-use_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -use_end"); + return false; + } + // pass through NULL as this is an optional field + if (use_endStr) { + use_end = psTimeFromISO(use_endStr, PS_TIME_UTC); + if (!use_end) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", use_endStr); + return false; + } + } else { + use_end = NULL; + } + } + + psString reduction = psMetadataLookupStr(&status, config->args, "-reduction"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -reduction"); + return false; + } + + psString label = psMetadataLookupStr(&status, config->args, "-label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label"); + return false; + } + + // we have to support multipe exp_tags + psMetadataItem *item = psMetadataLookup(config->args, "-exp_tag"); + if (!item) { + // this shouldn't actually happen when using psArgs + psError(PS_ERR_UNKNOWN, true, "-exp_tag is required"); + return false; + } + psMetadata *where = psMetadataAlloc(); + + // make sure that -exp_tag was parsed correctly + // XXX this can be removed someday + if (item->type == PS_DATA_METADATA_MULTI) { + psListIterator *iter = psListIteratorAlloc(item->data.list, 0, false); + psMetadataItem *mItem = NULL; + while ((mItem = psListGetAndIncrement(iter))) { + psString exp_tag = mItem->data.V; + // if exp_tag is NULL then it means that -exp_tag has not been + // specified + if (!exp_tag) { + psError(PS_ERR_UNKNOWN, true, + "at least one -exp_tag is required"); + psFree(where); + return false; + } + + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_tag", + PS_META_DUPLICATE_OK, "==", exp_tag)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_tag"); + psFree(iter); + psFree(where); + return false; + } + } + psFree(iter); + } else { + psAbort( "-exp_tag was not parsed correctly (this should not happen"); + } + + if (psListLength(where->list) < 1) { + psFree(where); + where = NULL; + } + + // check that the specified exp_tags actually exist + psArray *detrendExps = rawExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + if (!detrendExps) { + psError(PS_ERR_UNKNOWN, false, "no rawExp rows found"); + return false; + } + + // we should have one rawExp row per exp_tag specified + if (psListLength(item->data.list) != psArrayLength(detrendExps)) { + psAbort( "an -exp_tag matched more then one rawExp (this should not happen"); + + } + + // check to see if -filelevel was set on the command line + if (!filelevel) { + filelevel = psStringCopy(((rawExpRow *)detrendExps->data[0])->filelevel); + } + + // start a transaction so we don't end up with childlessed det_ids + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detrendExps); + return false; + } + + // the first iteration is always 0 + // XXX the camera name is set from the first inputExp + // XXX det_id + detRunInsert(config->dbh, + 0, + 0, + det_type, + mode, + "run", + filelevel, + workdir, + camera, + telescope, + exp_type, + reduction, + filter, + airmass_min, + airmass_max, + exp_time_min, + exp_time_max, + ccd_temp_min, + ccd_temp_max, + posang_min, + posang_max, + registered, + time_begin, + time_end, + use_begin, + use_end, + solang_min, + solang_max, + label, + 0 // parent + ); + psFree(registered); + psFree(time_begin); + psFree(time_end); + psFree(use_begin); + psFree(use_end); + long det_id = psDBLastInsertID(config->dbh); + + // create new detInputExp row(s) from the rawExp row(s) + psArray *inputExps = psArrayAllocEmpty(psArrayLength(detrendExps)); + for (long i = 0; i < psArrayLength(detrendExps); i++) { + detInputExpRow *inputExp = rawDetrenTodetInputExpRow( + detrendExps->data[i], + det_id, + 0 // the first iteration is explicitly 0 + ); + psArrayAdd(inputExps, 0, inputExp); + psFree(inputExp); + } + + psFree(detrendExps); + + // insert detInputExp objects into the database + if (!detInputExpInsertObjects(config->dbh, inputExps)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(inputExps); + return false; + } + psFree(inputExps); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // print the new det_id + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS32(where, PS_LIST_TAIL, "det_id", 0, "==", det_id); + detRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check results + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %ld(this should not happen)", det_id); + return false; + } + + if (!convertIdToStr(detRuns)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(detRuns); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, detRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detRuns); + return false; + } + psFree(detRuns); + + return true; +} + +#if 0 +// This function is used to convert the det_id from an int, as it is used +// internally, to be a string for external use. The rational being that we may +// want to change how det_id is generated in the future and don't want to +// external programs to become depending on this value being an int. +static bool convertDetIdToStr(psArray *mds) +{ + PS_ASSERT_PTR_NON_NULL(mds, false); + + for (long i = 0; i < psArrayLength(mds); i++) { + psMetadata *md = mds->data[i]; + bool status = false; + psS32 det_id = psMetadataLookupS32(&status, md, "det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for det_id"); + return false; + } + psMetadataRemoveKey(md, "det_id"); + psString det_idStr = psDBIntToString((psU64)det_id); + psMetadataAddStr(mds->data[i], PS_LIST_HEAD, "det_id", 0, NULL, det_idStr); + psFree(det_idStr); + } + + return true; +} +#endif + +static bool definebyqueryMode(pxConfig *config) +{ + bool status = false; + + PS_ASSERT_PTR_NON_NULL(config, false); + + // what type of detRun is this? + // det_type is required + psString det_type = psMetadataLookupStr(&status, config->args, "-det_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_type"); + return false; + } + if (!det_type) { + psError(PS_ERR_UNKNOWN, true, "-det_type is required"); + return false; + } + + // optional + psString filelevel = psMetadataLookupStr(&status, config->args, "-filelevel"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filelevel"); + return false; + } + + psString workdir = psMetadataLookupStr(&status, config->args, "-workdir"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -workdir"); + return false; + } + if (!workdir) { + psError(PS_ERR_UNKNOWN, true, "-workdir is required"); + return false; + } + psString camera = psMetadataLookupStr(&status, config->args, "-inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst"); + return false; + } + if (!camera) { + psError(PS_ERR_UNKNOWN, false, "-inst is required"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + { + bool status = false; + psString exp_type = psMetadataLookupStr(&status, config->args, "-select_exp_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_exp_type"); + return false; + } + if (exp_type) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_type", 0, "==", exp_type)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_type"); + psFree(where); + return false; + } + } + + // map -inst -> camera + psString camera = psMetadataLookupStr(&status, config->args, "-select_inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_inst"); + return false; + } + if (camera) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "camera", 0, "==", camera)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item inst"); + psFree(where); + return false; + } + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-select_telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_telescope"); + return false; + } + if (telescope) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "telescope", 0, "==", telescope)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item telescope"); + psFree(where); + return false; + } + } + + psString filter = psMetadataLookupStr(&status, config->args, "-select_filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_filter"); + return false; + } + if (filter) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "filter", 0, "==", filter)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item filter"); + psFree(where); + return false; + } + } + + psString uri = psMetadataLookupStr(&status, config->args, "-select_uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_uri"); + return false; + } + if (uri) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "uri", 0, "==", uri)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item uri"); + psFree(where); + return false; + } + } + + psString dateobs_begin = psMetadataLookupStr(&status, config->args, "-select_dateobs_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_dateobs_begin"); + psFree(where); + return false; + } + if (dateobs_begin) { + psTime *time = psTimeFromISO(dateobs_begin, PS_TIME_UTC); + if (!time) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", dateobs_begin); + psFree(time); + psFree(where); + return false; + } + if (!psMetadataAddTime(where, PS_LIST_TAIL, "dateobs", PS_META_DUPLICATE_OK, ">=", time)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item dateobs"); + psFree(time); + psFree(where); + return false; + } + psFree(time); + } + + psString dateobs_end = psMetadataLookupStr(&status, config->args, "-select_dateobs_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_dateobs_end"); + psFree(where); + return false; + } + if (dateobs_end) { + psTime *time = psTimeFromISO(dateobs_end, PS_TIME_UTC); + if (!time) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", dateobs_end); + psFree(time); + psFree(where); + return false; + } + if (!psMetadataAddTime(where, PS_LIST_TAIL, "dateobs", PS_META_DUPLICATE_OK, "<", time)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item dateobs"); + psFree(time); + psFree(where); + return false; + } + psFree(time); + } + + /** selection based on airmass range **/ + psF32 select_airmass_min = psMetadataLookupF32(&status, config->args, "-select_airmass_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_airmass_min"); + psFree(where); + return false; + } + if (isfinite(select_airmass_min)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "airmass", PS_META_DUPLICATE_OK, ">=", select_airmass_min)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item airmass"); + psFree(where); + return false; + } + } + psF32 select_airmass_max = psMetadataLookupF32(&status, config->args, "-select_airmass_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_airmass_max"); + psFree(where); + return false; + } + if (isfinite(select_airmass_max)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "airmass", PS_META_DUPLICATE_OK, "<=", select_airmass_max)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item airmass"); + psFree(where); + return false; + } + } + + psF32 select_sat_pixel_frac_max = psMetadataLookupF32(&status, config->args, "-select_sat_pixel_frac_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_sat_pixel_frac_max"); + psFree(where); + return false; + } + if (isfinite(select_sat_pixel_frac_max)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "sat_pixel_frac", PS_META_DUPLICATE_OK, "<=", select_sat_pixel_frac_max)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item sat_pixel_frac"); + psFree(where); + return false; + } + } + + /** selection based on exp_time range **/ + psF32 select_exp_time_min = psMetadataLookupF32(&status, config->args, "-select_exp_time_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_exp_time_min"); + psFree(where); + return false; + } + if (isfinite(select_exp_time_min)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "exp_time", PS_META_DUPLICATE_OK, ">=", select_exp_time_min)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_time"); + psFree(where); + return false; + } + } + psF32 select_exp_time_max = psMetadataLookupF32(&status, config->args, "-select_exp_time_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_exp_time_max"); + psFree(where); + return false; + } + if (isfinite(select_exp_time_max)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "exp_time", PS_META_DUPLICATE_OK, "<=", select_exp_time_max)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_time"); + psFree(where); + return false; + } + } + + /** selection based on ccd_temp range **/ + psF32 select_ccd_temp_min = psMetadataLookupF32(&status, config->args, "-select_ccd_temp_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_ccd_temp_min"); + psFree(where); + return false; + } + if (isfinite(select_ccd_temp_min)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "ccd_temp", PS_META_DUPLICATE_OK, ">=", select_ccd_temp_min)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item ccd_temp"); + psFree(where); + return false; + } + } + psF32 select_ccd_temp_max = psMetadataLookupF32(&status, config->args, "-select_ccd_temp_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_ccd_temp_max"); + psFree(where); + return false; + } + if (isfinite(select_ccd_temp_max)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "ccd_temp", PS_META_DUPLICATE_OK, "<=", select_ccd_temp_max)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item ccd_temp"); + psFree(where); + return false; + } + } + + /** selection based on posang **/ + psF32 select_posang_min = psMetadataLookupF32(&status, config->args, "-select_posang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_posang_min"); + psFree(where); + return false; + } + if (isfinite(select_posang_min)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "posang", PS_META_DUPLICATE_OK, ">=", select_posang_min)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item posang"); + psFree(where); + return false; + } + } + psF32 select_posang_max = psMetadataLookupF32(&status, config->args, "-select_posang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_posang_max"); + psFree(where); + return false; + } + if (isfinite(select_posang_max)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "posang", PS_META_DUPLICATE_OK, "<=", select_posang_max)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item posang"); + psFree(where); + return false; + } + } + + /** selection based on solang **/ + psF32 select_solang_min = psMetadataLookupF32(&status, config->args, "-select_solang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_solang_min"); + psFree(where); + return false; + } + if (isfinite(select_solang_min)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "solang", PS_META_DUPLICATE_OK, ">=", select_solang_min)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item solang"); + psFree(where); + return false; + } + } + psF32 select_solang_max = psMetadataLookupF32(&status, config->args, "-select_solang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -select_solang_max"); + psFree(where); + return false; + } + if (isfinite(select_solang_max)) { + if (!psMetadataAddF32(where, PS_LIST_TAIL, "solang", PS_META_DUPLICATE_OK, "<=", select_solang_max)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item solang"); + psFree(where); + return false; + } + } + + } + + if (!psListLength(where->list)) { + psFree(where); + where = NULL; + } + + // there is some namespace overlap between the names of the fields we'd + // like to search by to setup a detrun and the names of the fields we'd + // like to assign values to so I've seperated them but prepending set- to + // the assigned values + + // optional + psString mode = psMetadataLookupStr(&status, config->args, "-mode"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -mode"); + return false; + } + // check mode + if (mode && !isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalud mode"); + return false; + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope"); + return false; + } + + psString filter = psMetadataLookupStr(&status, config->args, "-filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter"); + return false; + } + + psF32 airmass_min = psMetadataLookupF32(&status, config->args, "-airmass_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass_min"); + return false; + } + + psF32 airmass_max = psMetadataLookupF32(&status, config->args, "-airmass_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass_max"); + return false; + } + + psF32 exp_time_min = psMetadataLookupF32(&status, config->args, "-exp_time_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time_min"); + return false; + } + + psF32 exp_time_max = psMetadataLookupF32(&status, config->args, "-exp_time_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time_max"); + return false; + } + + psF32 ccd_temp_min = psMetadataLookupF32(&status, config->args, "-ccd_temp_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp_min"); + return false; + } + + psF32 ccd_temp_max = psMetadataLookupF32(&status, config->args, "-ccd_temp_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp_max"); + return false; + } + + psF64 posang_min = psMetadataLookupF32(&status, config->args, "-posang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang_min"); + return false; + } + + psF64 posang_max = psMetadataLookupF32(&status, config->args, "-posang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang_max"); + return false; + } + + psF64 solang_min = psMetadataLookupF32(&status, config->args, "-solang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -solang_min"); + return false; + } + + psF64 solang_max = psMetadataLookupF32(&status, config->args, "-solang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -solang_max"); + return false; + } + + psTime *registered = NULL; + { + psString registeredStr = psMetadataLookupStr(&status, config->args, "-registered"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -registered"); + return false; + } + // pass through NULL as this is an optional field + if (registeredStr) { + registered = psTimeFromISO(registeredStr, PS_TIME_UTC); + if (!registered) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", registeredStr); + return false; + } + } else { + registered = NULL; + } + } + + psTime *time_begin = NULL; + { + psString time_beginStr = psMetadataLookupStr(&status, config->args, "-time_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time_begin"); + return false; + } + // pass through NULL as this is an optional field + if (time_beginStr) { + time_begin = psTimeFromISO(time_beginStr, PS_TIME_UTC); + if (!time_begin) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", time_beginStr); + return false; + } + } else { + time_begin = NULL; + } + } + + psTime *time_end = NULL; + { + psString time_endStr = psMetadataLookupStr(&status, config->args, "-time_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time_end"); + return false; + } + // pass through NULL as this is an optional field + if (time_endStr) { + time_end = psTimeFromISO(time_endStr, PS_TIME_UTC); + if (!time_end) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", time_endStr); + return false; + } + } else { + time_end = NULL; + } + } + + psTime *use_begin = NULL; + { + psString use_beginStr = psMetadataLookupStr(&status, config->args, "-use_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -use_begin"); + return false; + } + // pass through NULL as this is an optional field + if (use_beginStr) { + use_begin = psTimeFromISO(use_beginStr, PS_TIME_UTC); + if (!use_begin) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", use_beginStr); + return false; + } + } else { + use_begin = NULL; + } + } + + psTime *use_end = NULL; + { + psString use_endStr = psMetadataLookupStr(&status, config->args, "-use_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -use_end"); + return false; + } + // pass through NULL as this is an optional field + if (use_endStr) { + use_end = psTimeFromISO(use_endStr, PS_TIME_UTC); + if (!use_end) { + psError(PS_ERR_UNKNOWN, false, "error in time format %s", use_endStr); + return false; + } + } else { + use_end = NULL; + } + } + + psString reduction = psMetadataLookupStr(&status, config->args, "-reduction"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -reduction"); + return false; + } + + psString label = psMetadataLookupStr(&status, config->args, "-label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label"); + return false; + } + + bool simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return false; + } + + bool pretend = psMetadataLookupBool(&status, config->args, "-pretend"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -pretend"); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return false; + } + + // search for rawExps with the specified options + psArray *detrendExps = rawExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + // make sure that we found at least one rawExp + if (!detrendExps) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return false; + } + if (!psArrayLength(detrendExps)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(detrendExps); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return true; + } + + // check to see if -filelevel was set on the command line + if (!filelevel) { + filelevel = psStringCopy(((rawExpRow *)detrendExps->data[0])->filelevel); + } + + if (pretend) { + // negative simple so the default is true + if (!rawExpPrintObjects(stdout, detrendExps, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detrendExps); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return false; + } + psFree(detrendExps); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return true; + } + + // start a transaction so we don't end up with childlessed det_ids + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detrendExps); + psFree(registered); + psFree(use_begin); + psFree(use_end); + return false; + } + + // the first iteration is always 0 + // XXX det_id + detRunInsert(config->dbh, + 0, // det_id + 0, // iteration + det_type, + mode, + "run", // state + filelevel, + workdir, + camera, + telescope, + "NA", + reduction, + filter, + airmass_min, + airmass_max, + exp_time_min, + exp_time_max, + ccd_temp_min, + ccd_temp_max, + posang_min, + posang_max, + registered, + time_begin, + time_end, + use_begin, + use_end, + solang_min, + solang_max, + label, + 0 // parent + ); + psFree(registered); + psFree(time_begin); + psFree(time_end); + psFree(use_end); + psFree(use_begin); + psFree(use_end); + long det_id = psDBLastInsertID(config->dbh); + + // create new detInputExp row(s) from the rawExp row(s) + psArray *inputExps = psArrayAllocEmpty(psArrayLength(detrendExps)); + for (long i = 0; i < psArrayLength(detrendExps); i++) { + detInputExpRow *inputExp = rawDetrenTodetInputExpRow( + detrendExps->data[i], + det_id, + 0 // the first iteration is explicitly 0 + ); + psArrayAdd(inputExps, 0, inputExp); + psFree(inputExp); + } + + psFree(detrendExps); + + // insert detInputExp objects into the database + if (!detInputExpInsertObjects(config->dbh, inputExps)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(inputExps); + return false; + } + psFree(inputExps); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + // print the new det_id + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS32(where, PS_LIST_TAIL, "det_id", 0, "==", det_id); + detRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check results + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %ld(this should not happen)", det_id); + return false; + } + + // convert det_id to a string externaly + if (!convertIdToStr(detRuns)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert det_id to a string"); + psFree(detRuns); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, detRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detRuns); + return false; + } + psFree(detRuns); + + return true; +} + +static bool definebydetrunMode(pxConfig *config) +{ + bool status = false; + + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id is the only required input + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + + // lookup the detRun that we will be basing this one on + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", (psS64)atoll(det_id)); + detRuns = detRunSelectRowObjects(config->dbh, where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + // sanity check the result... we should have only found one det_id + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 " (this should not happen)", (psS64)atoll(det_id)); + return false; // unreachable + } + + // pull the detRun object out the result array + detRunRow *detRun = psMemIncrRefCounter(detRuns->data[0]); + + // discard the resultarray + psFree(detRuns); + + // set the det_id to 0/NULL so the database can assign it + detRun->det_id = 0; + + // reset the iteration to 0 + detRun->iteration = 0; + + // reset the state to "run" + psFree(detRun->state); + detRun->state = psStringCopy("run"); + + // walk through the optional values and update the detRun as required + psString det_type = psMetadataLookupStr(&status, config->args, "-set_det_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_det_type"); + return false; + } + if (det_type) { + psFree(detRun->det_type); + detRun->det_type = psStringCopy(det_type); + } + + psString mode = psMetadataLookupStr(&status, config->args, "-set_mode"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_mode"); + return false; + } + // check mode + if (mode && !isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalud mode"); + return false; + } + if (mode) { + psFree(detRun->mode); + detRun->mode = psStringCopy(mode); + } + + psString camera = psMetadataLookupStr(&status, config->args, "-set_inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_inst"); + return false; + } + if (camera) { + psFree(detRun->camera); + detRun->camera = psStringCopy(camera); + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-set_telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_telescope"); + return false; + } + if (telescope) { + psFree(detRun->telescope); + detRun->telescope = psStringCopy(telescope); + } + + psString exp_type = psMetadataLookupStr(&status, config->args, "-set_exp_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_exp_type"); + return false; + } + if (exp_type) { + psFree(detRun->exp_type); + detRun->exp_type = psStringCopy(exp_type); + } + + psString filelevel = psMetadataLookupStr(&status, config->args, "-set_filelevel"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_filelevel"); + return false; + } + if (filelevel) { + psFree(detRun->filelevel); + detRun->filelevel = psStringCopy(filelevel); + } + + psString workdir = psMetadataLookupStr(&status, config->args, "-set_workdir"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_workdir"); + return false; + } + if (workdir) { + psFree(detRun->workdir); + detRun->workdir = psStringCopy(workdir); + } + + psString filter = psMetadataLookupStr(&status, config->args, "-set_filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_filter"); + return false; + } + if (filter) { + psFree(detRun->filter); + detRun->filter = psStringCopy(filter); + } + + psF32 airmass_min = psMetadataLookupF32(&status, config->args, "-set_airmass_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_airmass_min"); + return false; + } + if (!isnan(airmass_min)) { + detRun->airmass_min = airmass_min; + } + + psF32 airmass_max = psMetadataLookupF32(&status, config->args, "-set_airmass_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_airmass_max"); + return false; + } + if (!isnan(airmass_max)) { + detRun->airmass_max = airmass_max; + } + + psF32 exp_time_min = psMetadataLookupF32(&status, config->args, "-set_exp_time_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_exp_time_min"); + return false; + } + if (!isnan(exp_time_min)) { + detRun->exp_time_min = exp_time_min; + } + + psF32 exp_time_max = psMetadataLookupF32(&status, config->args, "-set_exp_time_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_exp_time_max"); + return false; + } + if (!isnan(exp_time_max)) { + detRun->exp_time_max = exp_time_max; + } + + psF32 ccd_temp_min = psMetadataLookupF32(&status, config->args, "-set_ccd_temp_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_ccd_temp_min"); + return false; + } + if (!isnan(ccd_temp_min)) { + detRun->ccd_temp_min = ccd_temp_min; + } + + psF32 ccd_temp_max = psMetadataLookupF32(&status, config->args, "-set_ccd_temp_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_ccd_temp_max"); + return false; + } + if (!isnan(ccd_temp_max)) { + detRun->ccd_temp_max = ccd_temp_max; + } + + psF64 posang_min = psMetadataLookupF32(&status, config->args, "-set_posang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_posang_min"); + return false; + } + if (!isnan(posang_min)) { + detRun->posang_min = posang_min; + } + + psF64 posang_max = psMetadataLookupF32(&status, config->args, "-set_posang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_posang_max"); + return false; + } + if (!isnan(posang_max)) { + detRun->posang_max = posang_max; + } + + psF64 solang_min = psMetadataLookupF32(&status, config->args, "-set_solang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_solang_min"); + return false; + } + if (!isnan(solang_min)) { + detRun->solang_min = solang_min; + } + + psF64 solang_max = psMetadataLookupF32(&status, config->args, "-set_solang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_solang_max"); + return false; + } + if (!isnan(solang_max)) { + detRun->solang_max = solang_max; + } + + psString label = psMetadataLookupStr(&status, config->args, "-set_label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_label"); + return false; + } + if (label) { + detRun->label = label; + } + + psTime *registered = NULL; + { + psString registeredStr = psMetadataLookupStr(&status, config->args, "-set_registered"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_registered"); + return false; + } + // pass through NULL as this is an optional field + if (registeredStr) { + psFree(detRun->registered); + registered = psTimeFromISO(registeredStr, PS_TIME_UTC); + detRun->registered = psMemIncrRefCounter(registered); + psFree(registered); + } + } + + psTime *time_begin = NULL; + { + psString time_beginStr = psMetadataLookupStr(&status, config->args, "-set_time_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_time_begin"); + return false; + } + // pass through NULL as this is an optional field + if (time_beginStr) { + psFree(detRun->time_begin); + time_begin = psTimeFromISO(time_beginStr, PS_TIME_UTC); + detRun->time_begin = psMemIncrRefCounter(time_begin); + psFree(time_begin); + } + } + + psTime *time_end = NULL; + { + psString time_endStr = psMetadataLookupStr(&status, config->args, "-set_time_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_time_end"); + return false; + } + // pass through NULL as this is an optional field + if (time_endStr) { + psFree(detRun->time_end); + time_end = psTimeFromISO(time_endStr, PS_TIME_UTC); + detRun->time_end = psMemIncrRefCounter(time_end); + psFree(time_end); + } + } + + psTime *use_begin = NULL; + { + psString use_beginStr = psMetadataLookupStr(&status, config->args, "-set_use_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_use_begin"); + return false; + } + // pass through NULL as this is an optional field + if (use_beginStr) { + psFree(detRun->use_begin); + use_begin = psTimeFromISO(use_beginStr, PS_TIME_UTC); + detRun->use_begin = psMemIncrRefCounter(use_begin); + psFree(use_begin); + } + } + + psTime *use_end = NULL; + { + psString use_endStr = psMetadataLookupStr(&status, config->args, "-set_use_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_use_end"); + return false; + } + // pass through NULL as this is an optional field + if (use_endStr) { + psFree(detRun->use_end); + use_end = psTimeFromISO(use_endStr, PS_TIME_UTC); + detRun->use_end = psMemIncrRefCounter(use_end); + psFree(use_end); + } + } + + psString reduction = psMetadataLookupStr(&status, config->args, "-set_reduction"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_reduction"); + return false; + } + if (reduction) { + psFree(detRun->reduction); + detRun->reduction = psStringCopy(reduction); + } + + // create a metadata to restrict detInputExp's be in in the specified range + + psMetadata *time_filter = psMetadataAlloc(); + + { + psString timeStr = psMetadataLookupStr(&status, config->args, "-filter_input_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter_input_begin"); + return false; + } + // pass through NULL as this is an optional field + if (timeStr) { + psTime *time = psTimeFromISO(timeStr, PS_TIME_UTC); + if (!psMetadataAddTime(time_filter, PS_LIST_TAIL, "dateobs", PS_META_DUPLICATE_OK, ">=", time)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item dateobs"); + psFree(time); + psFree(time_filter); + return false; + } + psFree(time); + } + } + + { + psString timeStr = psMetadataLookupStr(&status, config->args, "-filter_input_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter_input_end"); + return false; + } + // pass through NULL as this is an optional field + if (timeStr) { + psTime *time = psTimeFromISO(timeStr, PS_TIME_UTC); + if (!psMetadataAddTime(time_filter, PS_LIST_TAIL, "dateobs", PS_META_DUPLICATE_OK, "<", time)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item dateobs"); + psFree(time); + psFree(time_filter); + return false; + } + psFree(time); + } + } + + + // start a transaction so we don't end up with childlessed det_ids + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(time_filter); + psFree(detRun); + return false; + } + + if (!detRunInsertObject(config->dbh, detRun)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(time_filter); + psFree(detRun); + return false; + } + psFree(detRun); + + // get the det_id + long newDet_id = psDBLastInsertID(config->dbh); + + psString query = psStringCopy( + "INSERT INTO detInputExp" + " SELECT" + " %d," + " 0," + " detResidExp.exp_tag," + " detResidExp.accept" + " FROM detResidExp" + " JOIN rawExp" + " USING(exp_tag)" + " WHERE det_id = %d" + ); + + if (time_filter->list->n) { + psString whereClause = psDBGenerateWhereConditionSQL(time_filter, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(time_filter); + + if (!p_psDBRunQuery(config->dbh, query, (psS64)newDet_id, (psS64)atoll(det_id))) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // print the new det_id + psArray *newDetRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS32(where, PS_LIST_TAIL, "det_id", 0, "==", newDet_id); + newDetRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!newDetRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check the result... we should have only found one det_id + if (psArrayLength(newDetRuns) != 1) { + psAbort("found more then one detRun matching det_id %ld(this should not happen)", newDet_id); + return false; // unreachable + } + + // convert det_id to a string externaly + if (!convertIdToStr(detRuns)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(detRuns); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, newDetRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(newDetRuns); + return false; + } + psFree(newDetRuns); + + return true; +} + +static bool runsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // XXX fix the hard coding of the table name + psArray *runs = psDBSelectRows(config->dbh, "detRun", config->where, 0); + if (!runs) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(runs)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(runs); + return true; + } + + // convert det_id to a string externaly + if (!convertIdToStr(runs)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(runs); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, runs, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(runs); + return false; + } + + psFree(runs); + + return true; +} + +static bool childlessrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = psStringCopy( + "SELECT DISTINCT\n" + " detRun.*\n" + " FROM detRun\n" + " LEFT JOIN detRun as foo\n" + " ON foo.parent = detRun.det_id\n" + " WHERE\n" + " detRun.state = 'stop'\n" + " AND detRun.mode = 'master'\n" + " AND foo.det_id IS NULL\n" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detRun"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // convert det_id to a string externaly + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static detInputExpRow *rawDetrenTodetInputExpRow(rawExpRow *rawExp, psS32 det_id, psS32 iteration) +{ + PS_ASSERT_PTR_NON_NULL(rawExp, NULL); + + return detInputExpRowAlloc( + det_id, + iteration, + rawExp->exp_tag, + true // use + ); +} + +static bool inputMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // select detInputExp.* + // select rawExp.* + // by: + // exp_tag + + psString query = psStringCopy( + "SELECT DISTINCT *" + " FROM detInputExp" + " JOIN rawExp" + " USING(exp_tag)" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereSQL(config->where, "detInputExp"); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detInputExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool rawMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = psStringCopy( + "SELECT" + " detInputExp.det_id," + " detRun.det_type," + " rawImfile.*," + " rawExp.camera" + " FROM rawImfile" + " JOIN detInputExp" + " USING(exp_tag) " + " JOIN rawExp" + " USING(exp_tag) " + " JOIN detRun" + " ON detInputExp.det_id = detRun.det_id" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool toprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + // select det_id, det_type & rawImfile * + // get det_id from detInputExp.det_id + // get det_type from detRun.det_id + // + // det_id is in detInputExp + // exp_tag is in detInputExp + // det_id is not in detProccessedImfile + // exp_tag is not in detProccessedImfile + // class_is is not in detProccessedImfile + psString query = psStringCopy( + "SELECT" + " detRun.det_id," + " detRun.det_type," + " detRun.workdir," + " detRun.reduction," + " rawImfile.*," + " rawExp.camera" + " FROM detRun" + " JOIN detInputExp" + " USING(det_id, iteration)" + " JOIN rawExp" + " ON detInputExp.exp_tag = rawExp.exp_tag" + " JOIN rawImfile" + " ON detInputExp.exp_tag = rawImfile.exp_tag" + " LEFT JOIN detProcessedImfile" + " ON detInputExp.det_id = detProcessedImfile.det_id" + " AND rawImfile.exp_tag = detProcessedImfile.exp_tag" + " AND rawImfile.class_id = detProcessedImfile.class_id" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + " AND detProcessedImfile.det_id IS NULL" + " AND detProcessedImfile.exp_tag IS NULL" + " AND detProcessedImfile.class_id IS NULL" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingProcessedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static psArray *searchRawImfiles(pxConfig *config, psMetadata *where) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // use the default where if "where" is NULL + if (!where) { + where = config->where; + } + + // select exp_tags from detInputExp matching det_idp + // where query should be pre-generated + psArray *detInputExp = + detInputExpSelectRowObjects(config->dbh, where, 0); + if (!detInputExp) { + psError(PS_ERR_UNKNOWN, false, "no rawExp rows found"); + return NULL; + } + + // generate where query with just the exp_tags + psMetadata *where_exp_tags = psMetadataAlloc(); + for (long i = 0; i < psArrayLength(detInputExp); i++) { + detInputExpRow *row = detInputExp->data[i]; + if (!psMetadataAddStr(where_exp_tags, PS_LIST_TAIL, "exp_tag", + PS_META_DUPLICATE_OK, "==", row->exp_tag) + ) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_tag"); + psFree(detInputExp); + psFree(where_exp_tags); + return NULL; + } + } + psFree(detInputExp); + + // select rawImfiles with matching exp_tags + psArray *rawImfiles = + rawImfileSelectRowObjects(config->dbh, where_exp_tags, 0); + psFree(where_exp_tags); + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found"); + return NULL; + } + + return rawImfiles; +} + +static bool addprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id, exp_tag, class_id, uri, recipe, -bg, -bg_stdev + // are required + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + psString exp_tag = psMetadataLookupStr(&status, config->args, "-exp_tag"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_tag"); + return false; + } + if (!exp_tag) { + psError(PS_ERR_UNKNOWN, true, "-exp_tag is required"); + return false; + } + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + return false; + } + if (!class_id) { + psError(PS_ERR_UNKNOWN, true, "-class_id is required"); + return false; + } + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + return false; + } + if (!recipe) { + psError(PS_ERR_UNKNOWN, true, "-recip is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + //psError(PS_ERR_UNKNOWN, true, "-bg is required"); + //return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + + // optional + psF64 fringe_0 = psMetadataLookupF64(&status, config->args, "-fringe_0"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_0"); + return false; + } + psF64 fringe_1 = psMetadataLookupF64(&status, config->args, "-fringe_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_1"); + return false; + } + psF64 fringe_2 = psMetadataLookupF64(&status, config->args, "-fringe_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_2"); + return false; + } + + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + // find the matching rawImfile by exp_tag/class_id + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_tag", 0, "==", exp_tag)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_tag"); + psFree(where); + return false; + } + if (!psMetadataAddStr(where, PS_LIST_TAIL, "class_id", 0, "==", class_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class_id"); + psFree(where); + return false; + } + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + psArray *rawImfiles = rawImfileSelectRowObjects(config->dbh, where, 0); + psFree(where); + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found "); + return false; + } + + + // create a new detProcessedImfile object + detProcessedImfileRow *detRow = detProcessedImfileRowAlloc( + (psS32)atol(det_id), + exp_tag, + class_id, + uri, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + fringe_0, + fringe_1, + fringe_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + code + ); + psFree(rawImfiles); + + // insert the new row into the detProcessedImfile table + if (!detProcessedImfileInsertObject(config->dbh, detRow)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detRow); + return false; + } + + psFree(detRow); + + return true; +} + +static bool toprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + // select detProcessedImfile.det_id + // select detRun.iteration + // select detRun.det_type + // select detProcessedImfile.exp_tag + // by: + // find the current iteration bassed on det_id + // find all exp_tags in the current det_id/iteration from detInputExp + // find all rawImfiles in the current exp_tags + // compare to detProcessedImfiles by det_id/exp_tag + // found how many imfile there are in each class_id + // and: + // det_id is not in detProcessedExp; + // iteration is not in detProcessedExp; + + psString query = psStringCopy( + " SELECT DISTINCT" + " det_id," + " iteration," + " det_type," + " exp_tag," + " camera," + " workdir" + " FROM" + " (SELECT DISTINCT" + " detRun.det_id," + " detRun.iteration," + " detRun.det_type," + " detRun.workdir," + " detProcessedImfile.exp_tag," + " rawExp.camera," + " detProcessedImfile.class_id," + " rawExp.imfiles" + " FROM detRun" + " JOIN detInputExp" + " USING(det_id, iteration)" + " JOIN rawExp" + " ON detInputExp.exp_tag = rawExp.exp_tag" + " JOIN rawImfile" + " ON rawExp.exp_tag = rawImfile.exp_tag" + " LEFT JOIN detProcessedImfile" + " ON detRun.det_id = detProcessedImfile.det_id" + " AND detInputExp.exp_tag = detProcessedImfile.exp_tag" + " AND rawImfile.class_id = detProcessedImfile.class_id" + " LEFT JOIN detProcessedExp" + " ON detProcessedImfile.det_id = detProcessedExp.det_id" + " AND detProcessedImfile.exp_tag = detProcessedExp.exp_tag" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + " AND detProcessedImfile.fault = 0" + " AND detProcessedExp.det_id IS NULL" + " AND detProcessedExp.exp_tag IS NULL" + " AND detInputExp.include = 1" + " GROUP BY" + " rawExp.exp_tag," + " detRun.det_id" + " HAVING" + " COUNT(detProcessedImfile.class_id) = rawExp.imfiles" + " ) AS detProcessedExp" + ); + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id, exp_tag, recip, -bg, -bg_stdev + // are required + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + psString exp_tag = psMetadataLookupStr(&status, config->args, "-exp_tag"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_tag"); + return false; + } + if (!exp_tag) { + psError(PS_ERR_UNKNOWN, true, "-exp_tag is required"); + return false; + } + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + return false; + } + if (!recipe) { + psError(PS_ERR_UNKNOWN, true, "-recip is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + // optional + psF64 fringe_0 = psMetadataLookupF64(&status, config->args, "-fringe_0"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_0"); + return false; + } + psF64 fringe_1 = psMetadataLookupF64(&status, config->args, "-fringe_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_1"); + return false; + } + psF64 fringe_2 = psMetadataLookupF64(&status, config->args, "-fringe_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_2"); + return false; + } + + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + psString query = psStringCopy( + " SELECT DISTINCT" + " detProcessedImfile.det_id," + " detRun.iteration," + " detRun.det_type," + " detProcessedImfile.exp_tag" + " FROM detRun" + " JOIN detInputExp" + " ON detRun.det_id = detInputExp.det_id" + " AND detRun.iteration = detInputExp.iteration" + " JOIN rawExp" + " ON detInputExp.exp_tag = rawExp.exp_tag" + " JOIN detProcessedImfile" + " ON detInputExp.det_id = detProcessedImfile.det_id" + " AND detInputExp.exp_tag = detProcessedImfile.exp_tag" + " LEFT JOIN detProcessedExp" + " ON detInputExp.det_id = detProcessedExp.det_id" + " AND detProcessedImfile.exp_tag= detProcessedExp.exp_tag" + " LEFT JOIN rawImfile" + " ON detInputExp.exp_tag = rawImfile.exp_tag" + " AND detProcessedImfile.class_id = rawImfile.class_id" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + " AND detProcessedExp.det_id IS NULL" + " AND detProcessedExp.exp_tag IS NULL" + " AND detInputExp.include = 1" + " AND detRun.det_id = %s" + " AND detProcessedImfile.exp_tag = '%s'" + " GROUP BY" + " detProcessedImfile.class_id," + " rawImfile.class_id," + " detRun.det_id" + " HAVING" + " COUNT(detProcessedImfile.class_id) = COUNT(rawImfile.class_id)" + ); + + if (!p_psDBRunQuery(config->dbh, query, det_id, exp_tag)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + psFree(output); + + // create a new detProcessedImfile object + detProcessedExpRow *detRow = detProcessedExpRowAlloc( + (psS32)atol(det_id), + exp_tag, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + fringe_0, + fringe_1, + fringe_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + code + ); + + // insert the new row into the detProcessedImfile table + if (!detProcessedExpInsertObject(config->dbh, detRow)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detRow); + return false; + } + + psFree(detRow); + + return true; +} + +static bool processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = psStringCopy("SELECT * FROM detProcessedExp"); + + if (config->where) { + psString whereClause = psDBGenerateWhereSQL(config->where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detProcessedExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detProcessedExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool tostackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + // select detRun.iteration + // select detProcessedImfile.det_id + // select detProcessedImfile.class_id + // by: + // find the current iteration bassed on det_id + // find all exp_tags in the current det_id/iteration from detInputExp + // find all rawImfiles in the current exp_tags + // compare to detProcessedImfiles by det_id/exp_tag + // found how many imfile there are in each class_id + // and: + // det_id is not in detStackedImfile; + // iteration is not in detStackedImfile; + // class_id is not in detStackedImfile; + + psString query = psStringCopy( + " SELECT\n" + " detProcessedImfile.det_id,\n" + " detRun.iteration,\n" + " detRun.det_type,\n" + " detRun.workdir,\n" + " detRun.reduction,\n" + " detProcessedImfile.class_id,\n" + " rawExp.camera\n" + " FROM detRun\n" + " JOIN detInputExp\n" + " ON detRun.det_id = detInputExp.det_id\n" + " AND detRun.iteration = detInputExp.iteration\n" + " JOIN rawExp\n" + " ON detInputExp.exp_tag = rawExp.exp_tag\n" + " JOIN rawImfile\n" + " ON detInputExp.exp_tag = rawImfile.exp_tag\n" + " LEFT JOIN detProcessedImfile\n" + " ON detInputExp.det_id = detProcessedImfile.det_id\n" + " AND detInputExp.exp_tag = detProcessedImfile.exp_tag\n" + " AND rawImfile.class_id = detProcessedImfile.class_id\n" + " LEFT JOIN detStackedImfile\n" + " ON detInputExp.det_id = detStackedImfile.det_id\n" + " AND detInputExp.iteration = detStackedImfile.iteration\n" + " AND rawImfile.class_id = detStackedImfile.class_id\n" + " WHERE\n" + " detRun.state = 'run'\n" + " AND detRun.mode = 'master'\n" + " AND (detProcessedImfile.fault = 0 or detProcessedImfile.fault IS NULL)\n" + " AND detStackedImfile.det_id IS NULL\n" + " AND detStackedImfile.iteration IS NULL\n" + " AND detStackedImfile.class_id IS NULL\n" + " AND detInputExp.include = 1\n" + " GROUP BY\n" + " rawImfile.class_id,\n" + " detRun.det_id\n" + " HAVING\n" + " COUNT(detProcessedImfile.class_id) = COUNT(rawImfile.class_id)\n" + ); + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingStackedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + char *value = NULL; + bool status = false; + + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = psStringCopy( + " SELECT DISTINCT" + " detRun.det_type," + " detProcessedImfile.*" + " FROM detProcessedImfile" + " JOIN detRun" + " USING(det_id)" + " JOIN detInputExp" + " ON detRun.det_id = detInputExp.det_id" + " AND detRun.iteration = detInputExp.iteration" + " AND detProcessedImfile.exp_tag = detInputExp.exp_tag" + " WHERE" + ); + // NOTE the above WHERE is completed with the following line: + + // add the two required restrictions: detRun.state and detRun.mode + if ((value = psMetadataLookupStr(&status, config->args, "-select_state"))) { + psStringAppend(&query, " detRun.state = '%s'", value); + } else { + psStringAppend(&query, " detRun.state = 'run'"); + } + if ((value = psMetadataLookupStr(&status, config->args, "-select_mode"))) { + psStringAppend(&query, " AND detRun.mode = '%s'", value); + } else { + psStringAppend(&query, " AND detRun.mode = 'master'"); + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detProcessedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + { + bool status = false; + bool included = psMetadataLookupBool(&status, config->args, "-included"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + // restrict search to included imfiles + if (included) { + psStringAppend(&query, " AND detInputExp.include = 1"); + } + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detProcessedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detProcessedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addstackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id, class_id, uri, & recipe are required + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + return false; + } + if (!class_id) { + psError(PS_ERR_UNKNOWN, true, "-class_id is required"); + return false; + } + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + return false; + } + if (!recipe) { + psError(PS_ERR_UNKNOWN, true, "-recip is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + + // optional values + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + // default values + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + return false; + } + + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + // correlate the class_id against the input exposure(s) + + // we have to generate our own where clause as we want to search only by the + // det_id + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddS32(where, PS_LIST_TAIL, "det_id", 0, "==", + (psS64)atoll(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_tag"); + psFree(where); + return false; + } + + psArray *rawImfiles = searchRawImfiles(config, where); + psFree(where); + + bool valid_class_id = false; + if (rawImfiles) { + for (long i = 0; i < psArrayLength(rawImfiles); i++) { + if (strcmp(class_id, ((rawImfileRow *)rawImfiles->data[i])->class_id) == 0) { + valid_class_id = true; + break; + } + } + psFree(rawImfiles); + } + + if (!valid_class_id) { + psError(PS_ERR_UNKNOWN, true, + "class_id can not be correlated with the input exposures"); + return false; + } + + // create a new detStackedImfile object + detStackedImfileRow *stackedImfile = detStackedImfileRowAlloc( + (psS32)atol(det_id), + iteration, + class_id, + uri, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + code + ); + + // insert the new row into the detProcessedImfile table + if (!detStackedImfileInsertObject(config->dbh, stackedImfile)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(stackedImfile); + return false; + } + + psFree(stackedImfile); + + return true; +} + +static bool stackedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + // select detStackedImfile.* + // by: + // where det_id, iteration, class_id is not in detNormalizedImfile + + psString query = psStringCopy( + "SELECT" + " detStackedImfile.*" + " FROM detStackedImfile" + " JOIN detRun" + " USING(det_id, iteration)" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detStackedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detStackedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detStackedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool tonormalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + // select detRun.det_id (det_id) + // select detRun.iteration + // by: + // find the current iteration bassed on det_id + // find all exp_tags in the current det_id/iteration from detInputExp + // sort to detInputExp.imfiles to find the largest value per det_id/iter + // compare imfiles to the number of detStackedImfiles by class_id + // and: + // ??? + // det_id is not in detStackedImfile; + // iteration is not in detStackedImfile; + // class_id is not in detStackedImfile; + + psString query = psStringCopy( + "SELECT" + " det_id," + " det_type," + " iteration," + " camera," + " workdir" + " FROM" + "(SELECT DISTINCT" + " detRun.det_id," + " detRun.det_type," + " detRun.iteration," + " detRun.workdir," + " rawExp.camera," + " rawExp.imfiles," + " detStackedImfile.class_id" + " FROM detRun" + " JOIN detInputExp" + " ON detRun.det_id = detInputExp.det_id" + " AND detRun.iteration = detInputExp.iteration" + " JOIN rawExp" + " ON detInputExp.exp_tag = rawExp.exp_tag" + " JOIN detStackedImfile" + " ON detInputExp.det_id = detStackedImfile.det_id" + " AND detInputExp.iteration = detStackedImfile.iteration" + " LEFT JOIN detNormalizedStatImfile" + " ON detStackedImfile.det_id = detNormalizedStatImfile.det_id" + " AND detStackedImfile.iteration = detNormalizedStatImfile.iteration" + " AND detStackedImfile.class_id = detNormalizedStatImfile.class_id" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + " AND detStackedImfile.fault = 0" + " AND detNormalizedStatImfile.det_id IS NULL" + " AND detNormalizedStatImfile.iteration IS NULL" + " AND detNormalizedStatImfile.class_id IS NULL" + " GROUP BY" + " rawExp.exp_tag," + " detRun.iteration," + " detRun.det_id" + " HAVING MAX(rawExp.imfiles) = COUNT(detStackedImfile.class_id)" + ") as tonormalizedstat" + ); + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingNormStatImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addnormalizedstatMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // select * from detStackedImfile + // by det_id, iteration, class_id + // where det_id, iteration, class_id is not in detNormalizedStatImfile + psString query = psStringCopy( + "SELECT DISTINCT" + " detStackedImfile.*" + " FROM detStackedImfile" + " LEFT JOIN detNormalizedStatImfile" + " USING(det_id, iteration, class_id)" + " WHERE" + " detNormalizedStatImfile.det_id IS NULL" + " AND detNormalizedStatImfile.iteration IS NULL" + " AND detNormalizedStatImfile.class_id IS NULL" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detStackedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + // convert metadata into a detStackedImfile object + detStackedImfileRow *stackedImfile = detStackedImfileObjectFromMetadata(row); + // convert detStackedImfile object into a detNormalizedStat object + detNormalizedStatImfileRow *stat = detStackedToDetNormalizedStatImfile(config, stackedImfile); + psFree(stackedImfile); + if (!stat) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to convert detStackedImfile to detNormalizedStatImfile"); + psFree(output); + return false; + } + // insert detNormlized Stat object into the database + if (!detNormalizedStatImfileInsertObject(config->dbh, stat)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(stat); + psFree(output); + } + psFree(stat); + } + + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static detNormalizedStatImfileRow *detStackedToDetNormalizedStatImfile(pxConfig *config, detStackedImfileRow *stackedImfile) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(stackedImfile, NULL); + + bool status = false; + psF32 norm = psMetadataLookupF32(&status, config->args, "-norm"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -norm"); + return false; + } + //if (isnan(norm)) { + // psError(PS_ERR_UNKNOWN, true, "-norm is required"); + // return false; + //} + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + return detNormalizedStatImfileRowAlloc( + stackedImfile->det_id, + stackedImfile->iteration, + stackedImfile->class_id, + norm, + code + ); +} + +static bool tonormalizeMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + // select detNormalizedStatImfile.* + // by: + // where det_id, iteration, class_id is not in detNormalizedImfile + + psString query = psStringCopy( + "SELECT DISTINCT" + " detRun.det_type," + " detRun.workdir," + " rawExp.camera," + " detStackedImfile.uri," + " detNormalizedStatImfile.*" + " FROM detRun" + " JOIN detStackedImfile" + " USING(det_id, iteration)" + " JOIN detInputExp" + " USING(det_id, iteration)" + " JOIN rawExp" + " ON detInputExp.exp_tag = rawExp.exp_tag" + " JOIN detNormalizedStatImfile" + " ON detStackedImfile.det_id = detNormalizedStatImfile.det_id" + " AND detStackedImfile.iteration = detNormalizedStatImfile.iteration" + " AND detStackedImfile.class_id = detNormalizedStatImfile.class_id" + " LEFT JOIN detNormalizedImfile" + " ON detNormalizedStatImfile.det_id = detNormalizedImfile.det_id" + " AND detNormalizedStatImfile.iteration = detNormalizedImfile.iteration" + " AND detNormalizedStatImfile.class_id = detNormalizedImfile.class_id" + " WHERE" + " detNormalizedImfile.det_id IS NULL" + " AND detNormalizedImfile.iteration IS NULL" + " AND detNormalizedImfile.class_id IS NULL" + " AND detNormalizedStatImfile.fault = 0" + ); + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingNormImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +#if 0 +// XXX this function was left in commented as this method may be useful in the +// future +static psArray *validDetInputClassIds(pxConfig *config, const char *det_id) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + // det_id is input as a string because the fact that it is an integer + // is just a database impliementation detail. + PS_ASSERT_PTR_NON_NULL(det_id, NULL); + + psArray *rawImfiles = searchInputImfiles(config, det_id); + if (!rawImfiles) { + return NULL; + } + + psArray *valid_class_ids = NULL; + { + // All this jumping through hoops is so that we end up with unique + // values. PP thinks that making multiple passes through this array and + // deleting matched elements would end up being more expensive then a + // double sort and stagger scheme. JH thinks it would be cheaper to + // just do a unqiue sort and delete. So this is really just a cheap + // hack to avoid implimenting a unique sort function but at least it's + // stable. + psHash *hash = psHashAlloc(psArrayLength(rawImfiles)); + for (long i = 0; i < psArrayLength(rawImfiles); i++) { + psHashAdd(hash, + ((rawImfileRow *)rawImfiles->data[i])->class_id, + ((rawImfileRow *)rawImfiles->data[i])->class_id + ); + } + valid_class_ids = psHashToArray(hash); + psFree(hash); + } + psFree(rawImfiles); + + return valid_class_ids; +} + +static psArray *searchInputImfiles(pxConfig *config, const char *det_id) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + // det_id is input as a string because the fact that it is an integer + // is just a database impliementation detail. + PS_ASSERT_PTR_NON_NULL(det_id, NULL); + + psArray *inputExps = NULL; + { + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddS32(where, PS_LIST_TAIL, "det_id", 0, "==", + (psS32)atoi(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return NULL; + } + inputExps = detInputExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + } + if (!inputExps) { + psError(PS_ERR_UNKNOWN, false, "no detInputExp rows found"); + return NULL; + } + + // find rawImfiles associated with detInputExps + psArray *rawImfiles = NULL; + { + psMetadata *where = psMetadataAlloc(); + for (long i = 0; i < psArrayLength(inputExps); i++) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_tag", + PS_META_DUPLICATE_OK, "==", + ((detInputExpRow *)inputExps->data[i])->exp_tag)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_tag"); + psFree(inputExps); + psFree(where); + return NULL; + } + } + psFree(inputExps); + rawImfiles = rawImfileSelectRowObjects(config->dbh, where, 0); + // XXX this really should be sorted for uniqueness + psFree(where); + } + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found"); + return NULL; + } + + return rawImfiles; +} +#endif + +static bool addnormalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // make sure that there is a respondoing entry in detNormalizedStatImfile + // select * from detNormalizedStatImfile + // by det_id, iteration, class_id + // where det_id, iteration, class_id is not in detNormalizedImfile + psString query = psStringCopy( + "SELECT" + " detNormalizedStatImfile.*" + " FROM detNormalizedStatImfile" + " LEFT JOIN detNormalizedImfile" + " USING(det_id, iteration, class_id)" + " WHERE" + " detNormalizedImfile.det_id IS NULL" + " AND detNormalizedImfile.iteration IS NULL" + " AND detNormalizedImfile.class_id IS NULL" + ); + + { + // build a query to search by det_id, iteration, class_id + psMetadata *where = psMetadataAlloc(); + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + psFree(where); + psFree(query); + return false; + } + if (det_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "det_id", 0, "==", det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + psFree(query); + return false; + } + } + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + psFree(where); + psFree(query); + return false; + } + // always set iteration + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", iteration)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + psFree(query); + return false; + } + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + psFree(where); + psFree(query); + return false; + } + if (class_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "class_id", 0, "==", class_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class_id"); + psFree(where); + psFree(query); + return false; + } + } + + // there's not + psString whereClause = psDBGenerateWhereConditionSQL(where, "detNormalizedStatImfile"); + psFree(where); + if (whereClause) { + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + // convert metadata into a detNormalizedStatImfile object + detNormalizedStatImfileRow *statImfile = detNormalizedStatImfileObjectFromMetadata(row); + // convert detNormalizedStatImfile object into a detNormalizedImfile + detNormalizedImfileRow *normalizedImfile = detNormalizedStatToDetNormalizedmfile(config, statImfile); + psFree(statImfile); + if (!normalizedImfile) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to convert detStackedImfile to detNormalizedStatImfile"); + psFree(output); + return false; + } + // insert detNormlized Stat object into the database + if (!detNormalizedImfileInsertObject(config->dbh, normalizedImfile)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(normalizedImfile); + psFree(output); + } + psFree(normalizedImfile); + } + + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool tonormalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = psStringCopy( + " SELECT DISTINCT" + " detRun.det_id as det_id," + " detRun.iteration," + " detRun.det_type," + " detRun.workdir," + " rawExp.camera," + " rawExp.telescope," + " rawExp.exp_type," + " rawExp.imfiles" + " FROM detRun" + " JOIN detInputExp" + " ON detRun.det_id = detInputExp.det_id" + " AND detRun.iteration = detInputExp.iteration" + " JOIN rawExp" + " ON detInputExp.exp_tag = rawExp.exp_tag" + " JOIN detNormalizedImfile" + " ON detInputExp.det_id = detNormalizedImfile.det_id" + " AND detInputExp.iteration = detNormalizedImfile.iteration" + " LEFT JOIN rawImfile" + " ON detInputExp.exp_tag = rawImfile.exp_tag" + " AND detNormalizedImfile.class_id = rawImfile.class_id" + " LEFT JOIN detNormalizedExp" + " ON detInputExp.det_id = detNormalizedExp.det_id" + " AND detInputExp.iteration = detNormalizedExp.iteration" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + " AND detNormalizedImfile.fault = 0" + " AND detNormalizedExp.det_id IS NULL" + " AND detNormalizedExp.iteration IS NULL" + " AND detInputExp.include = 1" + " GROUP BY" + " detNormalizedImfile.iteration," + " detRun.det_id" + " HAVING" + " COUNT(detNormalizedImfile.class_id) = COUNT(rawImfile.class_id)" + ); + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingNormExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addnormalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id, recip, -bg, -bg_stdev + // are required + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + return false; + } + if (!recipe) { + psError(PS_ERR_UNKNOWN, true, "-recip is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + + // optional + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + // iteration has a default value + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); return false; + } + + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + // optional + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + psString query = psStringCopy( + " SELECT DISTINCT" + " detRun.det_id as det_id," + " detRun.iteration," + " detRun.det_type," + " rawExp.camera," + " rawExp.telescope," + " rawExp.exp_type," + " rawExp.imfiles" + " FROM detRun" + " JOIN detInputExp" + " ON detRun.det_id = detInputExp.det_id" + " AND detRun.iteration = detInputExp.iteration" + " JOIN rawExp" + " ON detInputExp.exp_tag = rawExp.exp_tag" + " JOIN detNormalizedImfile" + " ON detInputExp.det_id = detNormalizedImfile.det_id" + " AND detInputExp.iteration = detNormalizedImfile.iteration" + " LEFT JOIN rawImfile" + " ON detInputExp.exp_tag = rawImfile.exp_tag" + " AND detNormalizedImfile.class_id = rawImfile.class_id" + " LEFT JOIN detNormalizedExp" + " ON detInputExp.det_id = detNormalizedExp.det_id" + " AND detInputExp.iteration = detNormalizedExp.iteration" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + " AND detNormalizedExp.det_id IS NULL" + " AND detNormalizedExp.iteration IS NULL" + " AND detInputExp.include = 1" + " AND detRun.det_id = %s" + " AND detNormalizedImfile.iteration = %d" + " GROUP BY" + " detNormalizedImfile.iteration," + " detRun.det_id" + " HAVING" + " COUNT(detNormalizedImfile.class_id) = COUNT(rawImfile.class_id)" + ); + + if (!p_psDBRunQuery(config->dbh, query, det_id, iteration)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + psFree(output); + + // create a new detProcessedImfile object + detNormalizedExpRow *detRow = detNormalizedExpRowAlloc( + (psS32)atol(det_id), + iteration, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + code + ); + + // insert the new row into the detProcessedImfile table + if (!detNormalizedExpInsertObject(config->dbh, detRow)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(detRow); + return false; + } + + psFree(detRow); + + return true; +} + +static bool normalizedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = psStringCopy( + "SELECT" + " detNormalizedExp.*" + " FROM detNormalizedExp" + " JOIN detRun" + " USING(det_id, iteration)" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereSQL(config->where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detNormalizedExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detNormalizedExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detNormalizedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static detNormalizedImfileRow *detNormalizedStatToDetNormalizedmfile(pxConfig *config, detNormalizedStatImfileRow *statImfile) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(statImfile, NULL); + + bool status = false; + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + // optional + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + // optional + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + return detNormalizedImfileRowAlloc( + statImfile->det_id, + statImfile->iteration, + statImfile->class_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + code + ); +} + +static bool toresidimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + /* +which returns a list of processed imfiles (with corresponding detrend +id, iteration, class id, uri, type) for which the appropriate stacked +imfile has been normalised, and which have not been masked out by +detResidImfileAnalysis; +*/ + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + psString query = psStringCopy( + "SELECT DISTINCT\n" + " detRun.det_id,\n" + " detRun.iteration,\n" + " detRun.det_type,\n" + " detRun.mode,\n" + " detRun.workdir,\n" + " detRun.reduction,\n" + " detProcessedImfile.exp_tag,\n" + " detProcessedImfile.class_id,\n" + " detProcessedImfile.uri,\n" + " detNormalizedImfile.uri AS det_uri,\n" + " rawExp.camera\n" + " FROM detRun\n" + " JOIN detInputExp\n" + " USING(det_id, iteration)\n" + " JOIN rawExp\n" + " ON detInputExp.exp_tag = rawExp.exp_tag\n" + " JOIN detProcessedImfile\n" + " ON detRun.det_id = detProcessedImfile.det_id\n" + " AND detInputExp.exp_tag = detProcessedImfile.exp_tag\n" + " JOIN detNormalizedImfile\n" + " ON detRun.det_id = detNormalizedImfile.det_id\n" + " AND detRun.iteration = detNormalizedImfile.iteration\n" + " AND detProcessedImfile.class_id = detNormalizedImfile.class_id\n" + " LEFT JOIN detResidImfile\n" + " ON detRun.det_id = detResidImfile.det_id\n" + " AND detRun.iteration = detResidImfile.iteration\n" + " AND detProcessedImfile.exp_tag = detResidImfile.exp_tag\n" + " AND detProcessedImfile.class_id = detResidImfile.class_id\n" + " WHERE\n" + " detRun.state = 'run'\n" + " AND detRun.mode = 'master'\n" + " AND detNormalizedImfile.fault = 0\n" + " AND detResidImfile.det_id IS NULL\n" + " AND detResidImfile.iteration IS NULL\n" + " AND detResidImfile.exp_tag IS NULL\n" + " AND detResidImfile.class_id IS NULL\n" + " UNION" + " SELECT\n" + " detRun.det_id,\n" + " detRun.iteration,\n" + " detRun.det_type,\n" + " detRun.mode,\n" + " detRun.workdir,\n" + " detRun.reduction,\n" + " rawImfile.exp_tag,\n" + " rawImfile.class_id,\n" + " rawImfile.uri,\n" + " 'NULL' AS det_uri,\n" + " rawExp.camera\n" + " FROM detRun\n" + " JOIN detInputExp\n" + " USING(det_id, iteration)\n" + " JOIN rawExp\n" + " ON detInputExp.exp_tag = rawExp.exp_tag\n" + " JOIN rawImfile\n" + " ON detInputExp.exp_tag = rawImfile.exp_tag\n" + " LEFT JOIN detResidImfile\n" + " ON detRun.det_id = detResidImfile.det_id\n" + " AND detRun.iteration = detResidImfile.iteration\n" + " AND rawImfile.exp_tag = detResidImfile.exp_tag\n" + " AND rawImfile.class_id = detResidImfile.class_id\n" + " WHERE\n" + " detRun.state = 'run'\n" + " AND detRun.mode = 'verify'\n" + " AND detResidImfile.det_id IS NULL\n" + " AND detResidImfile.iteration IS NULL\n" + " AND detResidImfile.exp_tag IS NULL\n" + " AND detResidImfile.class_id IS NULL\n" + ); + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingResidImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool normalizedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = psStringCopy( + "SELECT" + " detNormalizedImfile.*" + " FROM detNormalizedImfile" + " JOIN detRun" + " USING(det_id, iteration)" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + ); + + if (config->where) { + bool status; + int iteration = psMetadataLookupS32 (&status, config->where, "iteration"); + if (status) { + psMetadataRemoveKey (config->where, "iteration"); + psMetadataAddS32 (config->where, PS_LIST_TAIL, "iteration", 0, "==", iteration); + } + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detNormalizedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detNormalizedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detNormalizedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detNormalizedImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addresidimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // make sure that there is a coresponding entry in detNormalizedImfile + // and the exp_tag specified is valid + // select * from detNormalizedImfile + // by det_id, iteration, class_id + // where det_id, iteration, class_id is not in detResidImfile + psString query = psStringCopy( + " SELECT * FROM (" + "SELECT\n" + " detNormalizedImfile.*\n" + " FROM detRun\n" + " JOIN detInputExp\n" + " USING(det_id, iteration)\n" + " JOIN detNormalizedImfile\n" + " USING(det_id, iteration)\n" + " LEFT JOIN detResidImfile\n" + " ON detRun.det_id = detResidImfile.det_id\n" + " AND detRun.iteration = detResidImfile.iteration\n" + " AND detInputExp.exp_tag = detResidImfile.exp_tag\n" + " AND detNormalizedImfile.class_id = detResidImfile.class_id\n" + " WHERE\n" + " detRun.state = 'run'\n" + " AND detRun.mode= 'master'\n" + " AND detResidImfile.det_id IS NULL\n" + " AND detResidImfile.iteration IS NULL\n" + " AND detResidImfile.exp_tag IS NULL\n" + " AND detResidImfile.class_id IS NULL\n" + " UNION\n" + " SELECT\n" + " detRun.det_id,\n" + " detRun.iteration,\n" + " rawImfile.class_id,\n" + " rawImfile.uri,\n" + " 0.0 as bg,\n" + " 0.0 as bg_stdev,\n" + " 0.0 as bg_mean_stdev,\n" + " 0.0 as user_1,\n" + " 0.0 as user_2,\n" + " 0.0 as user_3,\n" + " 0.0 as user_4,\n" + " 0.0 as user_5,\n" + " 'NULL' as path_base,\n" + " 0 as fault\n" + " FROM detRun\n" + " JOIN detInputExp\n" + " USING(det_id, iteration)\n" + " JOIN rawExp\n" + " ON detInputExp.exp_tag = rawExp.exp_tag\n" + " JOIN rawImfile\n" + " ON detInputExp.exp_tag = rawImfile.exp_tag\n" + " LEFT JOIN detResidImfile\n" + " ON detRun.det_id = detResidImfile.det_id\n" + " AND detRun.iteration = detResidImfile.iteration\n" + " AND rawImfile.exp_tag = detResidImfile.exp_tag\n" + " AND rawImfile.class_id = detResidImfile.class_id\n" + " WHERE\n" + " detRun.state = 'run'\n" + " AND detRun.mode = 'verify'\n" + " AND detResidImfile.det_id IS NULL\n" + " AND detResidImfile.iteration IS NULL\n" + " AND detResidImfile.exp_tag IS NULL\n" + " AND detResidImfile.class_id IS NULL\n" + " AND detInputExp.exp_tag = '%1$s'\n" + ") AS foo\n" + " %2$s" + ); + + // build a query to search by det_id, iteration, class_id + psMetadata *where = psMetadataAlloc(); + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + psFree(where); + psFree(query); + return false; + } + if (det_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "det_id", 0, "==", det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + psFree(query); + return false; + } + } + + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + psFree(where); + psFree(query); + return false; + } + // always set iteration + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", iteration)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + psFree(query); + return false; + } + + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + psFree(where); + psFree(query); + return false; + } + if (class_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "class_id", 0, "==", class_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class_id"); + psFree(where); + psFree(query); + return false; + } + } + + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psFree(where); + + // exp_tag is manadatory to cross check that this is a exp_tag for this + // detRun + psString exp_tag = psMetadataLookupStr(&status, config->args, "-exp_tag"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_tag"); + psFree(whereClause); + psFree(query); + return false; + } + if (!exp_tag) { + psError(PS_ERR_UNKNOWN, true, "-exp_tag is required"); + psFree(whereClause); + psFree(query); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query, exp_tag, whereClause)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereClause); + psFree(query); + return false; + } + psFree(whereClause); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, false, "no pending detNormalizedImfile rows found"); + psFree(output); + return false; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + // convert metadata into a detNormalizedImfile object + detNormalizedImfileRow *normalizedImfile = detNormalizedImfileObjectFromMetadata(row); + // convert detNormalizedImfile object into a detResidImfile + detResidImfileRow *residImfile = detNormalizedToDetResidImfile(config, normalizedImfile); + psFree(normalizedImfile); + if (!residImfile) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to convert detNormalizedImfile to detResidImfile"); + psFree(output); + return false; + } + // insert detResidImfile object into the database + if (!detResidImfileInsertObject(config->dbh, residImfile)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(residImfile); + psFree(output); + } + psFree(residImfile); + } + + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static detResidImfileRow *detNormalizedToDetResidImfile(pxConfig *config, detNormalizedImfileRow *normalizedImfile) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(normalizedImfile, NULL); + + bool status = false; + psString exp_tag = psMetadataLookupStr(&status, config->args, "-exp_tag"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_tag"); + return false; + } + if (!exp_tag) { + psError(PS_ERR_UNKNOWN, true, "-exp_tag is required"); + return false; + } + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + return false; + } + if (!recipe) { + psError(PS_ERR_UNKNOWN, true, "-recip is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + psF64 bin_stdev = psMetadataLookupF64(&status, config->args, "-bin_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bin_stdev"); + return false; + } + + // optional + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + // optional + psF64 fringe_0 = psMetadataLookupF64(&status, config->args, "-fringe_0"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_0"); + return false; + } + psF64 fringe_1 = psMetadataLookupF64(&status, config->args, "-fringe_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_1"); + return false; + } + psF64 fringe_2 = psMetadataLookupF64(&status, config->args, "-fringe_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_2"); + return false; + } + + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + // create a new detResidImfileRow and insert it + return detResidImfileRowAlloc( + normalizedImfile->det_id, + normalizedImfile->iteration, + exp_tag, + normalizedImfile->class_id, + uri, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + bin_stdev, + fringe_0, + fringe_1, + fringe_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + code + ); +} + +static bool residimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + // select detResidImfile.* + // by: + // where det_id, iteration, exp_tag is not in detResidExp; + + psString query = psStringCopy( + "SELECT" + " detRun.det_type," + " detRun.mode," + " detResidImfile.*," + " rawExp.exp_time" + " FROM detResidImfile" + " JOIN detRun" + " USING(det_id, iteration)" + " JOIN rawExp" + " USING(exp_tag)" + " WHERE" + " detRun.state = 'run'" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detResidImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detResidImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detResidImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawResidImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool toresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + /* +which returns a list of exposures for which all component class ids +have had residuals created. This list includes the detrend id, +iteration, exposure id, detrend type, and whether the exposure was +included in the stack for this iteration. + */ + + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + // select detRun.det_id + // select detRun.iteration + // select detRun.det_type + // select detInputExp.exp_tag + // select detInputExp.include + // by: + // find the current iteration bassed on det_id + // find all exp_tags in the current det_id/iteration from detInputExp + // compare to detInputExp.imfiles to derResidImfile by class_id + // and: + // detResidImfile.{det_id, iteration, exp_tag} is not in detResidExp + + psString query = psStringCopy( + "SELECT DISTINCT\n" + " det_id,\n" + " iteration,\n" + " det_type,\n" + " mode,\n" + " exp_tag,\n" + " include,\n" + " camera,\n" + " workdir\n" + " FROM\n" + " (SELECT DISTINCT\n" + " detRun.det_id AS det_id,\n" + " detRun.iteration,\n" + " detRun.det_type,\n" + " detRun.mode,\n" + " detRun.workdir,\n" + " detInputExp.exp_tag,\n" + " detInputExp.include,\n" + " rawExp.imfiles,\n" + " rawExp.camera,\n" + " detResidImfile.class_id\n" + " FROM detRun\n" + " JOIN detInputExp\n" + " USING(det_id, iteration)\n" + " JOIN rawExp\n" + " ON detInputExp.exp_tag = rawExp.exp_tag\n" + " JOIN detResidImfile\n" + " ON detRun.det_id = detResidImfile.det_id\n" + " AND detRun.iteration = detResidImfile.iteration\n" + " AND detInputExp.exp_tag = detResidImfile.exp_tag\n" + " LEFT JOIN detResidExp\n" + " ON detResidImfile.det_id = detResidExp.det_id\n" + " AND detResidImfile.iteration = detResidExp.iteration\n" + " AND detResidImfile.exp_tag = detResidExp.exp_tag\n" + " WHERE\n" + " detRun.state = 'run'\n" + " AND detResidImfile.fault = 0\n" + " AND detResidExp.det_id IS NULL\n" + " AND detResidExp.iteration IS NULL\n" + " AND detResidExp.exp_tag IS NULL\n" + " GROUP BY\n" + " detInputExp.exp_tag,\n" + " detRun.iteration,\n" + " detRun.det_id\n" + " HAVING\n" + " rawExp.imfiles = COUNT(detResidImfile.class_id)\n" + " ) AS toresidexp\n" + ); + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detPendingResidExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + /* +which returns a list of exposures for which all component class ids +have had residuals created. This list includes the detrend id, +iteration, exposure id, detrend type, and whether the exposure was +included in the stack for this iteration. + */ + + + // select detRun.det_id + // select detRun.iteration + // select detRun.det_type + // select detInputExp.exp_tag + // select detInputExp.include + // by: + // find the current iteration bassed on det_id + // find all exp_tags in the current det_id/iteration from detInputExp + // compare to detInputExp.imfiles to derResidImfile by class_id + // and: + // detResidImfile.{det_id, iteration, exp_tag} is not in detResidExp + + psString query = psStringCopy( + "SELECT DISTINCT" + " det_id," + " iteration," + " exp_tag," + " include" + " FROM" + " (SELECT DISTINCT" + " detRun.det_id AS det_id," + " detRun.iteration," + " detRun.det_type," + " detInputExp.exp_tag," + " detInputExp.include," + " rawExp.imfiles" + " FROM detRun" + " JOIN detInputExp" + " USING(det_id, iteration)" + " JOIN rawExp" + " ON detInputExp.exp_tag = rawExp.exp_tag" + " JOIN detResidImfile" + " ON detRun.det_id = detResidImfile.det_id" + " AND detRun.iteration = detResidImfile.iteration" + " AND detInputExp.exp_tag = detResidImfile.exp_tag" + " LEFT JOIN detResidExp" + " ON detResidImfile.det_id = detResidExp.det_id" + " AND detResidImfile.iteration = detResidExp.iteration" + " AND detResidImfile.exp_tag = detResidExp.exp_tag" + " WHERE" + " detRun.state = 'run'" + " AND detResidExp.det_id IS NULL" + " AND detResidExp.iteration IS NULL" + " AND detResidExp.exp_tag IS NULL" + " GROUP BY" + " detInputExp.exp_tag," + " detRun.iteration," + " detRun.det_id" + " HAVING" + " rawExp.imfiles = COUNT(detResidImfile.class_id)" + " ) AS toresidexp" + ); + + { + // build a query to search by det_id, iteration, exp_tag + psMetadata *where = psMetadataAlloc(); + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + if (!psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", (psS64)atoll(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + psFree(query); + return false; + } + + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + psFree(where); + psFree(query); + return false; + } + // always set iteration + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", iteration)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + psFree(query); + return false; + } + psString exp_tag = psMetadataLookupStr(&status, config->args, "-exp_tag"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_tag"); + psFree(where); + psFree(query); + return false; + } + if (exp_tag) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_tag", 0, "==", exp_tag)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_tag"); + psFree(where); + psFree(query); + return false; + } + } + + // there's not + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psFree(where); + if (whereClause) { + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no detResidImfile rows found"); + psFree(output); + return false; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + // convert metadata into a detResidExp object + detResidExpRow *residExp = mdToDetResidExp(config, row); + if (!residExp) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata to detResidExp"); + psFree(output); + return false; + } + // insert detResidExp object into the database + if (!detResidExpInsertObject(config->dbh, residExp)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(residExp); + psFree(output); + } + psFree(residExp); + } + + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static detResidExpRow *mdToDetResidExp(pxConfig *config, psMetadata *row) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(row, NULL); + + bool status = false; + // values from row + psS32 det_id = psMetadataLookupS32(&status, row, "det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for det_id"); + return false; + } + //if (isnan(det_id)) { + // psError(PS_ERR_UNKNOWN, true, "det_id is required"); + // return false; + //} + psS32 iteration = psMetadataLookupS32(&status, row, "iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for iteration"); + return false; + } + psString exp_tag = psMetadataLookupStr(&status, row, "exp_tag"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_tag"); + return false; + } + if (!exp_tag) { + psError(PS_ERR_UNKNOWN, true, "exp_tag is required"); + return false; + } + // values from config + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + return false; + } + if (!recipe) { + psError(PS_ERR_UNKNOWN, true, "-recip is required"); + return false; + } + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return false; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return false; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + psF64 bin_stdev = psMetadataLookupF64(&status, config->args, "-bin_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bin_stdev"); + return false; + } + // optional + psF64 fringe_0 = psMetadataLookupF64(&status, config->args, "-fringe_0"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_0"); + return false; + } + psF64 fringe_1 = psMetadataLookupF64(&status, config->args, "-fringe_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_1"); + return false; + } + psF64 fringe_2 = psMetadataLookupF64(&status, config->args, "-fringe_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -fringe_2"); + return false; + } + + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + // optional + bool reject = psMetadataLookupBool(&status, config->args, "-reject"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -reject"); + return false; + } + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + if (!path_base) { + psError(PS_ERR_UNKNOWN, true, "-path_base is required"); + return false; + } + + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + // create a new detResidImfileRow and insert it + return detResidExpRowAlloc( + det_id, + iteration, + exp_tag, + recipe, + bg, + bg_stdev, + bg_mean_stdev, + bin_stdev, + fringe_0, + fringe_1, + fringe_2, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + !reject, + code + ); +} + +static bool residexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = psStringCopy( + "SELECT" + " detRun.mode," + " detResidExp.*," + " detInputExp.include" + " FROM detRun" + " JOIN detInputExp" + " USING(det_id, iteration)" + " JOIN detResidExp" + " USING(det_id, iteration, exp_tag)" + " WHERE" + " detRun.state = 'run'" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detResidExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detResidExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detResidExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detResidExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool residdetrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + /* which returns a list of detrend runs (with detrend id, iteration and + * detrend type) which have completed all residexps. + */ + + // select detRun.det_id + // select detRun.iteration + // select detRun.det_type + // by: + // find the current iteration bassed on det_id + // find all exp_tags in the current det_id/iteration from detInputExp + // find all exp_tags in the current det_id/iteration from detResidExp + // compare the counts of exp_tags + + psString query = psStringCopy( + "SELECT DISTINCT\n" + " det_id,\n" + " iteration,\n" + " det_type,\n" + " mode,\n" + " workdir,\n" + " camera\n" + " FROM\n" + " (SELECT DISTINCT\n" + " detRun.det_id,\n" + " detRun.iteration,\n" + " detRun.det_type,\n" + " detRun.mode,\n" + " detRun.workdir,\n" + " detInputExp.exp_tag,\n" + " rawExp.camera\n" + " FROM detRun\n" + " JOIN detInputExp\n" + " USING(det_id, iteration)\n" + " JOIN rawExp\n" + " ON detInputExp.exp_tag = rawExp.exp_tag\n" + " LEFT JOIN detResidExp\n" + " ON detRun.det_id = detResidExp.det_id\n" + " AND detRun.iteration = detResidExp.iteration\n" + " AND detInputExp.exp_tag = detResidExp.exp_tag\n" + " WHERE\n" + " detRun.state = 'run'\n" + " GROUP BY\n" + " detRun.det_id,\n" + " detRun.iteration\n" + " HAVING\n" + " COUNT(detResidExp.exp_tag) = COUNT(detInputExp.exp_tag)\n" + " ) AS residdetrun\n" + ); + + // XXX does it make sense to accept any search params? +#if 0 + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } +#endif + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detRejectExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool updateresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // build a query to search by det_id, iteration, exp_tag + psMetadata *where = psMetadataAlloc(); + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + psFree(where); + return false; + } + if (det_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "det_id", 0, "==", det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return false; + } + } + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + psFree(where); + return false; + } + // always where iteration + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", iteration)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + return false; + } + psString exp_tag = psMetadataLookupStr(&status, config->args, "-exp_tag"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_tag"); + psFree(where); + return false; + } + if (exp_tag) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_tag", 0, "==", exp_tag)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_tag"); + psFree(where); + return false; + } + } + + // find the values we're going to set + // copy everything but det_id, iteration, & exp_tag from the args and + // remove the '-' prefix + psMetadata *set = psMetadataAlloc(); + psString recipe = psMetadataLookupStr(&status, config->args, "-recip"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip"); + psFree(set); + psFree(where); + return false; + } + if (recipe) { + if (!psMetadataAddStr(set, PS_LIST_TAIL, "recipe", 0, "==", recipe)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item recipe"); + psFree(set); + psFree(where); + return false; + } + } + + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + psFree(set); + psFree(where); + return false; + } + if (!isnan(bg)) { + if (!psMetadataAddF64(set, PS_LIST_TAIL, "bg", 0, "==", bg)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item bg"); + psFree(set); + psFree(where); + return false; + } + } + + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + psFree(set); + psFree(where); + return false; + } + if (!isnan(bg_stdev)) { + if (!psMetadataAddF64(set, PS_LIST_TAIL, "bg_stdev", 0, "==", bg_stdev)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item bg_stdev"); + psFree(set); + psFree(where); + return false; + } + } + + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + psFree(set); + psFree(where); + return false; + } + if (!isnan(bg_mean_stdev)) { + if (!psMetadataAddF64(set, PS_LIST_TAIL, "bg_mean_stdev", 0, "==", bg_mean_stdev)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item bg_mean_stdev"); + psFree(set); + psFree(where); + return false; + } + } + + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + psFree(set); + psFree(where); + return false; + } + if (path_base) { + if (!psMetadataAddStr(set, PS_LIST_TAIL, "path_base", 0, "==", path_base)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item path_base"); + psFree(set); + psFree(where); + return false; + } + } + + bool reject = psMetadataLookupBool(&status, config->args, "-reject"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -reject"); + psFree(set); + psFree(where); + return false; + } + if (!psMetadataAddBool(set, PS_LIST_TAIL, "accept", 0, "==", !reject)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item accept"); + psFree(set); + psFree(where); + return false; + } + + long changed = psDBUpdateRows(config->dbh, "detResidExp", where, set); + psFree(set); + psFree(where); + + if (changed < 0) { + psError(PS_ERR_UNKNOWN, false, "no rows were updated"); + return false; + } + + return true; +} + +static bool adddetrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // select detRun.det_id + // select detRun.iteration + // by: + // find the current iteration bassed on det_id + // find all exp_tags in the current det_id/iteration from detInputExp + // find all exp_tags in the current det_id/iteration from detResidExp + // compare the counts of exp_tags + + psString query = pxDataGet("dettool_find_completed_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + { + // build a query to search by det_id, iteration, exp_tag + psMetadata *where = psMetadataAlloc(); + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + psFree(where); + psFree(query); + return false; + } + if (det_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "det_id", 0, "==", det_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + psFree(query); + return false; + } + } + psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration"); + psFree(where); + psFree(query); + return false; + } + // always set iteration + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", iteration)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + psFree(query); + return false; + } + + // there's not + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psFree(where); + if (whereClause) { + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + // convert metadata into a detResidExp object + detRunSummaryRow *runSummary = mdToDetRunSummary(config, row); + if (!runSummary) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata to detResidExp"); + psFree(output); + return false; + } + // insert detResidExp object into the database + if (!detRunSummaryInsertObject(config->dbh, runSummary)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(runSummary); + psFree(output); + return false; + } + psFree(runSummary); + } + + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; +} + +static detRunSummaryRow *mdToDetRunSummary(pxConfig *config, psMetadata *row) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + // from row + psS32 det_id = psMetadataLookupS32(&status, row, "det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for det_id"); + return false; + } + psS32 iteration = psMetadataLookupS32(&status, row, "iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for iteration"); + return false; + } + + // from config->args + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return NULL; + } + //if (isnan(bg)) { + // psError(PS_ERR_UNKNOWN, true, "-bg is required"); + // return NULL; + //} + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return NULL; + } + //if (isnan(bg_stdev)) { + // psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required"); + // return NULL; + //} + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return NULL; + } + // optional + bool accept = psMetadataLookupBool(&status, config->args, "-accept"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -accept"); + return NULL; + } + // default values + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + return detRunSummaryRowAlloc( + det_id, + iteration, + bg, + bg_stdev, + bg_mean_stdev, + accept, + code + ); +} + +static bool detrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = psStringCopy( + "SELECT DISTINCT\n" + " detRunSummary.*,\n" + " detRun.det_type,\n" + " detRun.mode\n" + " FROM detRun\n" + " JOIN detRunSummary\n" + " USING(det_id, iteration)\n" + " WHERE\n" + " detRun.state = 'run'\n" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detRunSummary.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detRunSummary.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawDetrendImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool updatedetrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id is required + // XXX this isn't strictly nessicary but not having the det_id complicates + // incrementing the iteration number + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + // either -rerun or -state must be specified + bool again = psMetadataLookupBool(&status, config->args, "-again"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -again"); + return false; + } + psString state = psMetadataLookupStr(&status, config->args, "-state"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -state"); + return false; + } + if (!(again || state)) { + psError(PS_ERR_UNKNOWN, true, "either -again or -state must be specified"); + return false; + } + if (again && state) { + psError(PS_ERR_UNKNOWN, true, "either -again or -state must be specified"); + return false; + } + + if (state) { + // set detRun.state to state + return setDetRunState(config, det_id, state); + } + + // else + // -again + + // select detRun.det_id + // select detRun.iteration + // select detInputExp.exp_tag + // select detResidExp.accept + // by: + // find the current iteration bassed on det_id + // find all exp_tags in the current det_id/iteration from detInputExp + // find all exp_tags in the current det_id/iteration from detResidExp + // compare the counts of exp_tags + + psString query = psStringCopy( + "SELECT DISTINCT" + " detRun.det_id AS det_id," + " detRun.iteration," + " detInputExp.exp_tag," + " detResidExp.accept" + " FROM detRun" + " JOIN detInputExp" + " ON detRun.det_id = detInputExp.det_id" + " AND detRun.iteration = detInputExp.iteration" + " JOIN detResidExp" + " ON detRun.det_id = detResidExp.det_id" + " AND detRun.iteration = detResidExp.iteration" + " AND detInputExp.exp_tag = detResidExp.exp_tag" + " WHERE" + " detRun.state = 'run'" + " AND detRun.mode = 'master'" + ); + + // XXX this query was not restricted by det_id, resulting + // in an inconsistent UPDATE below. I added this AND clause + // though there may be a cleaner method (EAM 2006.10.08) + psStringAppend ( + &query, + " AND detRun.det_id = '%s'", det_id); + + psStringAppend ( + &query, + " GROUP BY" + " detRun.det_id," + " detRun.iteration," + " detInputExp.exp_tag" + " HAVING" + " COUNT(detResidExp.exp_tag) = COUNT(detInputExp.exp_tag)" + ); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so we don't end up with an incremented iteration + // count but no detInputExps + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // up the detRuns iteration count for for the single det_id we are + // operating on + // XXX this will have to changed in order to support multiple det_ids with + // a single invocation of this functions + psS32 newIteration = incrementIteration(config, det_id); + if (!newIteration) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + psString exp_tag = psMetadataLookupStr(&status, row, "exp_tag"); + if (!status) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_tag"); + psFree(output); + return false; + } + bool accept = psMetadataLookupBool(&status, row, "accept"); + if (!status) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for accept"); + psFree(output); + return false; + } + + // detResidExp.include is used to set detInputExp.include + if (!detInputExpInsert( + config->dbh, + (psS32)atol(det_id), + newIteration, + exp_tag, + accept + ) + ) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + } + + psFree(output); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool rerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // det_id is required + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + + // we have to support multipe exp_tags + psMetadataItem *item = psMetadataLookup(config->args, "-exp_tag"); + if (!item) { + // this shouldn't actually happen when using psArgs + psError(PS_ERR_UNKNOWN, true, "-exp_tag is required"); + return false; + } + + psList *exp_tag_list = item->data.list; + psMetadata *where = psMetadataAlloc(); + // make sure that -exp_tag was parsed correctly + // XXX this can be removed someday + if (item->type == PS_DATA_METADATA_MULTI) { + psListIterator *iter = psListIteratorAlloc(item->data.list, 0, false); + psMetadataItem *mItem = NULL; + while ((mItem = psListGetAndIncrement(iter))) { + psString exp_tag = mItem->data.V; + // if exp_tag is NULL then it means that -exp_tag has not been + // specified + if (!exp_tag) { + psError(PS_ERR_UNKNOWN, true, + "at least one -exp_tag is required"); + psFree(where); + return false; + } + + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_tag", + PS_META_DUPLICATE_OK, "==", exp_tag)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_tag"); + psFree(iter); + psFree(where); + return false; + } + } + psFree(iter); + } else { + psAbort( "-exp_tag was not parsed correctly (this should not happen"); + } + + // check that the specified exp_tags actually exist in the iteration zero + // detInputExp set + + // add the det_id & iteration == 0 to the where clause + if (!psMetadataAddS32(where, PS_LIST_TAIL, "det_id", 0, "==", + (psS32)atol(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return false; + } + if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", 0)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(where); + return false; + } + + psArray *detrendExps = detInputExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + if (!detrendExps) { + psError(PS_ERR_UNKNOWN, false, "no rawExp rows found"); + psFree(where); + return false; + } + + // build a hash for the valid exp_tags + psHash *valid_exp_tags = psHashAlloc(psArrayLength(detrendExps)); + for (long i = 0; i < psArrayLength(detrendExps); i++) { + psHashAdd(valid_exp_tags, + ((detInputExpRow *)detrendExps->data[i])->exp_tag, + detrendExps->data[i] + ); + } + psFree(detrendExps); + + // start a transaction so we don't end up with an incremented iteration + // count but no detInputExps + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(valid_exp_tags); + return false; + } + + // up the detRuns iteration count + psS32 newIteration = incrementIteration(config, det_id); + if (!newIteration) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(valid_exp_tags); + return false; + } + + // check exp_tags and build up an array of new detInputExp rows at the same + // time + psListIterator *iter = psListIteratorAlloc(exp_tag_list, 0, false); + psMetadataItem *mItem = NULL; + psArray *newInputExps = psArrayAllocEmpty(psListLength(exp_tag_list)); + while ((mItem = psListGetAndIncrement(iter))) { + detInputExpRow *inputExp = psHashLookup(valid_exp_tags, + (char *)mItem->data.V); + if (!inputExp) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + // invalid exp_tag + psError(PS_ERR_UNKNOWN, false, "exp_tag %s is invalid for det_id %s", + (char *)mItem->data.V, det_id); + psFree(iter); + psFree(valid_exp_tags); + return false; + } + detInputExpRow *newInputExp = detInputExpRowAlloc( + (psS32)atol(det_id), + newIteration, + inputExp->exp_tag, + true // use + ); + psArrayAdd(newInputExps, 0, newInputExp); + psFree(newInputExp); + } + psFree(iter); + psFree(valid_exp_tags); + + for (long i = 0; i < psArrayLength(newInputExps); i++) { + if (!detInputExpInsertObject(config->dbh, newInputExps->data[i])) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(newInputExps); + return false; + } + } + psFree(newInputExps); + + // point of no return for det_id creation + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool register_detrendMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + bool status = false; + psString det_type = psMetadataLookupStr(&status, config->args, "-det_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_type"); + return false; + } + if (!det_type) { + psError(PS_ERR_UNKNOWN, true, "-det_type is required"); + return false; + } + + psString filelevel = psMetadataLookupStr(&status, config->args, "-filelevel"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filelevel"); + return false; + } + if (!filelevel) { + psError(PS_ERR_UNKNOWN, true, "-filelevel is required"); + return false; + } + + psString workdir = psMetadataLookupStr(&status, config->args, "-workdir"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -workdir"); + return false; + } + if (!workdir) { + psError(PS_ERR_UNKNOWN, true, "-workdir is required"); + return false; + } + + // everything else is optional + psString mode = psMetadataLookupStr(&status, config->args, "-mode"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -mode"); + return false; + } + // check mode + if (mode && !isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalud mode"); + return false; + } + + psString camera = psMetadataLookupStr(&status, config->args, "-inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst"); + return false; + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope"); + return false; + } + + psString exp_type = psMetadataLookupStr(&status, config->args, "-exp_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_type"); + return false; + } + + psString filter = psMetadataLookupStr(&status, config->args, "-filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter"); + return false; + } + + psF32 airmass_min = psMetadataLookupF32(&status, config->args, "-airmass_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass_min"); + return false; + } + + psF32 airmass_max = psMetadataLookupF32(&status, config->args, "-airmass_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass_max"); + return false; + } + + psF32 exp_time_min = psMetadataLookupF32(&status, config->args, "-exp_time_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time_min"); + return false; + } + + psF32 exp_time_max = psMetadataLookupF32(&status, config->args, "-exp_time_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time_max"); + return false; + } + + psF32 ccd_temp_min = psMetadataLookupF32(&status, config->args, "-ccd_temp_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp_min"); + return false; + } + + psF32 ccd_temp_max = psMetadataLookupF32(&status, config->args, "-ccd_temp_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp_max"); + return false; + } + + psF64 posang_min = psMetadataLookupF32(&status, config->args, "-posang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang_min"); + return false; + } + + psF64 posang_max = psMetadataLookupF32(&status, config->args, "-posang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang_max"); + return false; + } + + psF64 solang_min = psMetadataLookupF32(&status, config->args, "-solang_min"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -solang_min"); + return false; + } + + psF64 solang_max = psMetadataLookupF32(&status, config->args, "-solang_max"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -solang_max"); + return false; + } + + psTime *registered = NULL; + { + psString registeredStr = psMetadataLookupStr(&status, config->args, "-registered"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -registered"); + return false; + } + // pass through NULL as this is an optional field + if (registeredStr) { + registered = psTimeFromISO(registeredStr, PS_TIME_UTC); + } else { + registered = NULL; + } + } + + psTime *time_begin = NULL; + { + psString time_beginStr = psMetadataLookupStr(&status, config->args, "-time_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time_begin"); + return false; + } + // pass through NULL as this is an optional field + if (time_beginStr) { + time_begin = psTimeFromISO(time_beginStr, PS_TIME_UTC); + } else { + time_begin = NULL; + } + } + + psTime *time_end = NULL; + { + psString time_endStr = psMetadataLookupStr(&status, config->args, "-time_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -time_end"); + return false; + } + // pass through NULL as this is an optional field + if (time_endStr) { + time_end = psTimeFromISO(time_endStr, PS_TIME_UTC); + } else { + time_end = NULL; + } + } + + psTime *use_begin = NULL; + { + psString use_beginStr = psMetadataLookupStr(&status, config->args, "-use_begin"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -use_begin"); + return false; + } + // pass through NULL as this is an optional field + if (use_beginStr) { + use_begin = psTimeFromISO(use_beginStr, PS_TIME_UTC); + } else { + use_begin = NULL; + } + } + + psTime *use_end = NULL; + { + psString use_endStr = psMetadataLookupStr(&status, config->args, "-use_end"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -use_end"); + return false; + } + // pass through NULL as this is an optional field + if (use_endStr) { + use_end = psTimeFromISO(use_endStr, PS_TIME_UTC); + } else { + use_end = NULL; + } + } + + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + + psString parent = psMetadataLookupStr(&status, config->args, "-parent"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -parent"); + return false; + } + + psString label = psMetadataLookupStr(&status, config->args, "-label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!detRunInsert(config->dbh, + 0, // det_id + 0, // the iteration is fixed at 0 + det_type, + mode, + "reg", // state + filelevel, + workdir, + camera, + telescope, + exp_type, + NULL, + filter, + airmass_min, + airmass_max, + exp_time_min, + exp_time_max, + ccd_temp_min, + ccd_temp_max, + posang_min, + posang_max, + registered, + time_begin, + time_end, + use_begin, + use_end, + solang_min, + solang_max, + label, // label + parent ? (psS64)atoll(parent) : 0 + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(registered); + return false; + } + psFree(registered); + + if (!detRunSummaryInsert(config->dbh, + psDBLastInsertID(config->dbh), + 0, // the iteration is fixed at 0 + bg, + bg_stdev, + bg_mean_stdev, + true, // accept + 0 // fault code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // print the new detRun + psS64 det_id = psDBLastInsertID(config->dbh); + psArray *detRuns = NULL; + { + psMetadata *where = psMetadataAlloc(); + psMetadataAddS32(where, PS_LIST_TAIL, "det_id", 0, "==", det_id); + detRuns = psDBSelectRows(config->dbh, "detRun", where, 0); + psFree(where); + } + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "can't find the detRun we just created"); + return false; + } + // sanity check results + if (psArrayLength(detRuns) != 1) { + psAbort("found more then one detRun matching det_id %" PRId64 "(this should not happen)", det_id); + return false; + } + + if (!convertIdToStr(detRuns)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(detRuns); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, detRuns, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(detRuns); + return false; + } + psFree(detRuns); + + return true; +} + +static bool register_detrend_imfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + bool status = false; + psString det_id = psMetadataLookupStr(&status, config->args, "-det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id"); + return false; + } + if (!det_id) { + psError(PS_ERR_UNKNOWN, true, "-det_id is required"); + return false; + } + + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + return false; + } + if (!class_id) { + psError(PS_ERR_UNKNOWN, true, "-class_id is required"); + return false; + } + + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + + // everything else is optional + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + + // XXX this is going in without checking that the det_id is valid. + // it seems like it be better to be using foreign key constraints rather + // then having to resort to a seperate query (and locking) to check that + // the det_id is valid + if (!detNormalizedImfileInsert(config->dbh, + (psS64)atoll(det_id), + 0, // the iteration is fixed at 0 + class_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + 0 // fault code + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static psS32 incrementIteration(pxConfig *config, const char *det_id) +{ + // this function returns zero on error + PS_ASSERT_PTR_NON_NULL(config, 0); + PS_ASSERT_PTR_NON_NULL(det_id, 0); + + char *query = "UPDATE detRun SET iteration = iteration + 1 WHERE det_id = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to increment iteration for det_id %s", det_id); + return 0; + } + + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddS32(where, PS_LIST_TAIL, "det_id", 0, "==", + (psS32)atol(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return 0; + } + + psArray *detRuns = detRunSelectRowObjects(config->dbh, where, 0); + psFree(where); + if (!detRuns) { + psError(PS_ERR_UNKNOWN, false, "no detRun rows found"); + return 0; + } + + // sanity check the database + if (psArrayLength(detRuns) != 1) { + // this should no happen + psAbort( "database query return too many rows (this should not happen"); + } + + psS32 newIteration = ((detRunRow *)detRuns->data[0])->iteration; + psFree(detRuns); + + return newIteration; +} + +static bool setDetRunState(pxConfig *config, const char *det_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(det_id, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "drop", 5) == 0) + || (strncmp(state, "reg", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid detRun state: %s", state); + return false; + } + + char *query = "UPDATE detRun SET state = '%s' WHERE det_id = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, state, det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %s", det_id); + return false; + } + + return true; +} + +static bool isValidMode(pxConfig *config, const char *mode) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(mode, false); + + // check that state is a valid string value + if (!( + (strncmp(mode, "master", 7) == 0) + || (strncmp(mode, "verify", 7) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid detRun mode: %s", mode); + return false; + } + + return true; +} Index: /branches/jh_branches/backtrack/ippTools/src/dettool.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/dettool.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/dettool.h (revision 22250) @@ -0,0 +1,70 @@ +/* + * dettool.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef DETTOOL_H +#define DETTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + DETTOOL_MODE_NONE = PXTOOL_MODE_NONE, + DETTOOL_MODE_PENDING, + DETTOOL_MODE_DEFINEBYTAG, + DETTOOL_MODE_DEFINEBYQUERY, + DETTOOL_MODE_DEFINEBYDETRUN, + DETTOOL_MODE_RUNS, + DETTOOL_MODE_CHILDLESSRUN, + DETTOOL_MODE_INPUT, + DETTOOL_MODE_RAW, + DETTOOL_MODE_TOPROCESSEDIMFILE, + DETTOOL_MODE_ADDPROCESSEDIMFILE, + DETTOOL_MODE_PROCESSEDIMFILE, + DETTOOL_MODE_TOPROCESSEDEXP, + DETTOOL_MODE_ADDPROCESSEDEXP, + DETTOOL_MODE_PROCESSEDEXP, + DETTOOL_MODE_TOSTACKED, + DETTOOL_MODE_ADDSTACKED, + DETTOOL_MODE_STACKED, + DETTOOL_MODE_TONORMALIZE, + DETTOOL_MODE_ADDNORMALIZEDSTAT, + DETTOOL_MODE_TONORMALIZEDSTAT, + DETTOOL_MODE_ADDNORMALIZEDIMFILE, + DETTOOL_MODE_NORMALIZEDIMFILE, + DETTOOL_MODE_TONORMALIZEDEXP, + DETTOOL_MODE_ADDNORMALIZEDEXP, + DETTOOL_MODE_NORMALIZEDEXP, + DETTOOL_MODE_TORESIDIMFILE, + DETTOOL_MODE_ADDRESIDIMFILE, + DETTOOL_MODE_RESIDIMFILE, + DETTOOL_MODE_TORESIDEXP, + DETTOOL_MODE_RESIDEXP, + DETTOOL_MODE_ADDRESIDEXP, + DETTOOL_MODE_RESIDDETRUN, + DETTOOL_MODE_UPDATERESIDEXP, + DETTOOL_MODE_ADDDETRUNSUMMARY, + DETTOOL_MODE_DETRUNSUMMARY, + DETTOOL_MODE_UPDATEDETRUN, + DETTOOL_MODE_RERUN, + DETTOOL_MODE_REGISTER_DETREND, + DETTOOL_MODE_REGISTER_DETREND_IMFILE +} dettoolMode; + +pxConfig *dettoolConfig(pxConfig *config, int argc, char **argv); + +#endif // DETTOOL_H Index: /branches/jh_branches/backtrack/ippTools/src/dettoolConfig.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/dettoolConfig.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/dettoolConfig.c (revision 22250) @@ -0,0 +1,1069 @@ +/* + * dettoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "dettool.h" + +pxConfig *dettoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *time = psTimeGetNow(PS_TIME_TAI); + psString now = psTimeToISO(time); + psFree(time); + + // -pending + psMetadata *pendingArgs = psMetadataAlloc(); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search by exposure ID", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type", 0, + "search by exposure type", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst", 0, + "search by camera", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope", 0, + "search by telescope", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter", 0, + "search by filter", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-uri", 0, + "search by URL", NULL); + psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -definebytag + psMetadata *definebytagArgs = psMetadataAlloc(); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-exp_tag", + PS_META_DUPLICATE_OK, + "include this exposure (multiple OK, required)", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-det_type", 0, + "define the type of detrend run (required)", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-mode", 0, + "define the mode of this detrend run", "master"); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-filelevel", 0, + "define filelevel", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-workdir", 0, + "define workdir (required)", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-inst", 0, + "define camera", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-telescope", 0, + "define telescope", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-exp_type", 0, + "define exposure type", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-filter", 0, + "define filter ", NULL); + psMetadataAddF32(definebytagArgs, PS_LIST_TAIL, "-airmass_min", 0, + "define min airmass", NAN); + psMetadataAddF32(definebytagArgs, PS_LIST_TAIL, "-airmass_max", 0, + "define max airmass", NAN); + psMetadataAddF32(definebytagArgs, PS_LIST_TAIL, "-exp_time_min", 0, + "define min exposure time", NAN); + psMetadataAddF32(definebytagArgs, PS_LIST_TAIL, "-exp_time_max", 0, + "define max exposure time", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, + "define min ccd tempature", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, + "define max ccd tempature", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-posang_min", 0, + "define min rotator position angle", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-posang_max", 0, + "define max rotator position angle", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-solang_min", 0, + "define min solar angle", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-solang_max", 0, + "define max solar angle", NAN); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-registered", 0, + "time detrend run was registered", now); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-time_begin", 0, + "detrend applyes to exposures taken during this peroid", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-time_end", 0, + "detrend applyes to exposures taken during this peroid", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-use_begin", 0, + "start of detrend run applicable peroid", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-use_end", 0, + "end of detrend run applicable peroid", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-reduction", 0, + "define reduction class for processing", NULL); + psMetadataAddStr(definebytagArgs, PS_LIST_TAIL, "-label", 0, + "define detrun label", NULL); + psMetadataAddBool(definebytagArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-det_type", 0, + "define the type of detrend run (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-mode", 0, + "define the mode of this detrend run", "master"); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filelevel", 0, + "define filelevel", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, + "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-inst", 0, + "define camera (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-telescope", 0, + "define telescope", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filter", 0, + "define filter ", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-airmass_min", 0, + "define min airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-airmass_max", 0, + "define max airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_min", 0, + "define min exposure time", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_max", 0, + "define max exposure time", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, + "define min ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, + "define max ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_min", 0, + "define min rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_max", 0, + "define max rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-solang_min", 0, + "define min solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-solang_max", 0, + "define max solar angle", NAN); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, + "time detrend run was registered", now); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-time_begin", 0, + "detrend applyes to exposures taken during this peroid", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-time_end", 0, + "detrend applyes to exposures taken during this peroid", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-use_begin", 0, + "start of detrend run applicable peroid", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-use_end", 0, + "end of detrend run applicable peroid", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_type", 0, + "search for exp_type", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_inst", 0, + "search for camera", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_telescope", 0, + "search for telescope", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", 0, + "search for filter", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_uri", 0, + "search for uri", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_dateobs_begin", 0, + "search for exposures by time (>=)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_dateobs_end", 0, + "search for exposures by time (<)", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_airmass_min", 0, + "define min airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_airmass_max", 0, + "define max airmass", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_sat_pixel_frac_max", 0, + "define max fraction of saturated pixels", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_min", 0, + "define min exposure time", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_max", 0, + "define max exposure time", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_ccd_temp_min", 0, + "define min ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_ccd_temp_max", 0, + "define max ccd tempature", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_min", 0, + "define min rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_max", 0, + "define max rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_min", 0, + "define min solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_max", 0, + "define max solar angle", NAN); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, + "print the exposures that would be included in the detrend run and exit", false); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, + "define reduction class for processing", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, + "define detrun label", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -definebydetrun + psMetadata *definebydetrunArgs = psMetadataAlloc(); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-det_id", 0, + "det ID to base a new detRun on (required)", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_det_type", 0, + "define the type of detrend run", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_mode", 0, + "define the mode of this detrend run", "master"); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_exp_type", 0, + "define exposure type", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_filelevel", 0, + "define filelevel", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_workdir", 0, + "define workdir", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_filter", 0, + "define filter ", NULL); + psMetadataAddF32(definebydetrunArgs, PS_LIST_TAIL, "-set_airmass_min", 0, + "define airmass", NAN); + psMetadataAddF32(definebydetrunArgs, PS_LIST_TAIL, "-set_airmass_max", 0, + "define airmass", NAN); + psMetadataAddF32(definebydetrunArgs, PS_LIST_TAIL, "-set_exp_time_min", 0, + "define exposure time", NAN); + psMetadataAddF32(definebydetrunArgs, PS_LIST_TAIL, "-set_exp_time_max", 0, + "define exposure time", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_ccd_temp_min", 0, + "define ccd tempature", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_ccd_temp_max", 0, + "define ccd tempature", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_posang_min", 0, + "define rotator position angle", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_posang_max", 0, + "define rotator position angle", NAN); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_registered", 0, + "time detrend run was registered", now); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_time_begin", 0, + "start of peroid to apply detrend too", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_time_end", 0, + "end of peroid to apply detrend too", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_use_begin", 0, + "start of detrend run applicable peroid", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_use_end", 0, + "end of detrend run applicable peroid", NULL); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_solang_min", 0, + "define solar angle", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_solang_max", 0, + "define solar angle", NAN); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-filter_input_begin", 0, + "filter input detrun exp to be in this peroid", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-filter_input_end", 0, + "filter input detrun exp to be in this peroid", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_reduction", 0, + "define reduction class for processing", NULL); + psMetadataAddStr(definebydetrunArgs, PS_LIST_TAIL, "-set_label", 0, + "define detrun label", NULL); + psMetadataAddBool(definebydetrunArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -runs + psMetadata *runsArgs = psMetadataAlloc(); + psMetadataAddStr(runsArgs, PS_LIST_TAIL, "-det_type", 0, + "search for type of detrend run", NULL); + psMetadataAddBool(runsArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -childlessrun + psMetadata *childlessrunArgs = psMetadataAlloc(); + psMetadataAddStr(childlessrunArgs, PS_LIST_TAIL, "-det_type", 0, + "search for type of detrend run", NULL); + psMetadataAddU64(childlessrunArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(childlessrunArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -input + psMetadata *inputArgs = psMetadataAlloc(); + psMetadataAddStr(inputArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID", NULL); + psMetadataAddS32(inputArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddStr(inputArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search for exp ID", NULL); + psMetadataAddBool(inputArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -raw + psMetadata *rawArgs = psMetadataAlloc(); + psMetadataAddBool(rawArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -toprocessedimfile + psMetadata *toprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(toprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID", NULL); + psMetadataAddStr(toprocessedimfileArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search for exp ID", NULL); + psMetadataAddStr(toprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search for class ID", NULL); + psMetadataAddU64(toprocessedimfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(toprocessedimfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addprocessedimfile + psMetadata *addprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-exp_tag", 0, + "define exp ID (required)", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "define class ID (required)", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-uri", 0, + "define URI (required)", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-recip", 0, + "define recipe (required)", NULL); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background (required)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev (required)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_0", 0, + "define fringe slope (0th term)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_1", 0, + "define fringe slope (1st term)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-fringe_2", 0, + "define fringe slope (2nd term)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_1", 0, + "define user statistic (1)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_2", 0, + "define user statistic (2)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_3", 0, + "define user statistic (3)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_4", 0, + "define user statistic (4)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_5", 0, + "define user statistic (5)", NAN); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + + // -processedimfile + psMetadata *processedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search for exp ID", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search for class ID", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-select_state", 0, + "search for state", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-select_mode", 0, + "search for mode", NULL); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-chip", 0, + "restrict results to completed 'chip' sets", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-exp", 0, + "restrict results to complete 'exposures'", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-included", 0, + "restrict results to exposures 'includeded' in the current iteration", false); + psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -toprocessedexp + psMetadata *toprocessedexpArgs = psMetadataAlloc(); + psMetadataAddU64(toprocessedexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(toprocessedexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addproccessedexp + psMetadata *addprocessedexpArgs = psMetadataAlloc(); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-exp_tag", 0, + "define detrend ID (required)", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-recip", 0, + "define recipe (required)", NULL); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background (required)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev (required)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_0", 0, + "define fringe slope (0th term)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_1", 0, + "define fringe slope (1st term)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-fringe_2", 0, + "define fringe slope (2nd term)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_1", 0, + "define user statistic (1)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_2", 0, + "define user statistic (2)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_3", 0, + "define user statistic (3)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_4", 0, + "define user statistic (4)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_5", 0, + "define user statistic (5)", NAN); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + + // -proccessedexp + psMetadata *processedexpArgs = psMetadataAlloc(); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_tag", 0, + "define detrend ID", NULL); + psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -tostacked + psMetadata *tostackedArgs = psMetadataAlloc(); + psMetadataAddU64(tostackedArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(tostackedArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addstacked + psMetadata *addstackedArgs = psMetadataAlloc(); + psMetadataAddStr(addstackedArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddS32(addstackedArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddStr(addstackedArgs, PS_LIST_TAIL, "-class_id", 0, + "define class ID (required)", NULL); + psMetadataAddStr(addstackedArgs, PS_LIST_TAIL, "-recip", 0, + "define recipe (required)", NULL); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background (required)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev (required)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_1", 0, + "define user statistic (1)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_2", 0, + "define user statistic (2)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_3", 0, + "define user statistic (3)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_4", 0, + "define user statistic (4)", NAN); + psMetadataAddF64(addstackedArgs, PS_LIST_TAIL, "-user_5", 0, + "define user statistic (5)", NAN); + psMetadataAddStr(addstackedArgs, PS_LIST_TAIL, "-uri", 0, + "define URI (required)", NULL); + psMetadataAddS16(addstackedArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + + // -stacked + psMetadata *stackedArgs = psMetadataAlloc(); + psMetadataAddStr(stackedArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID", NULL); + psMetadataAddS32(stackedArgs, PS_LIST_TAIL, "-iteration", 0, + "search for iteration number", 0); + psMetadataAddStr(stackedArgs, PS_LIST_TAIL, "-class_id", 0, + "search for class ID", NULL); + psMetadataAddStr(stackedArgs, PS_LIST_TAIL, "-recip", 0, + "search for recipe", NULL); + psMetadataAddU64(stackedArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(stackedArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(stackedArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -tonormalize + psMetadata *tonormalizeArgs = psMetadataAlloc(); + psMetadataAddU64(tonormalizeArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(tonormalizeArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addnormalizedstat + psMetadata *addnormstatArgs = psMetadataAlloc(); + psMetadataAddStr(addnormstatArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddS32(addnormstatArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddStr(addnormstatArgs, PS_LIST_TAIL, "-class_id", 0, + "define class ID (required)", NULL); + psMetadataAddF32(addnormstatArgs, PS_LIST_TAIL, "-norm", 0, + "define normal value (required)", NAN); + psMetadataAddS16(addnormstatArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + + // -tonormstat + psMetadata *tonormstatArgs = psMetadataAlloc(); + psMetadataAddU64(tonormstatArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(tonormstatArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addnormalizedimfile + psMetadata *addnormalizedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(addnormalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddS32(addnormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddStr(addnormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "define class ID (required)", NULL); + psMetadataAddStr(addnormalizedimfileArgs, PS_LIST_TAIL, "-uri", 0, + "define URI (required)", NULL); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_1", 0, + "define user statistic (1)", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_2", 0, + "define user statistic (2)", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_3", 0, + "define user statistic (3)", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_4", 0, + "define user statistic (4)", NAN); + psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-user_5", 0, + "define user statistic (5)", NAN); + psMetadataAddStr(addnormalizedimfileArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + psMetadataAddS16(addnormalizedimfileArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + + // -tonormalizedexp + psMetadata *tonormalizedexpArgs = psMetadataAlloc(); + psMetadataAddU64(tonormalizedexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(tonormalizedexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addnormalizedexp + psMetadata *addnormalizedexpArgs = psMetadataAlloc(); + psMetadataAddStr(addnormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddS32(addnormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddStr(addnormalizedexpArgs, PS_LIST_TAIL, "-recip", 0, + "search for recipe (required)", NULL); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background (required)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev (required)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_1", 0, + "define user statistic (1)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_2", 0, + "define user statistic (2)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_3", 0, + "define user statistic (3)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_4", 0, + "define user statistic (4)", NAN); + psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-user_5", 0, + "define user statistic (5)", NAN); + + psMetadataAddStr(addnormalizedexpArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + psMetadataAddS16(addnormalizedexpArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + + // -normalizedexp + psMetadata *normalizedexpArgs = psMetadataAlloc(); + psMetadataAddStr(normalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID", NULL); + psMetadataAddS32(normalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddStr(normalizedexpArgs, PS_LIST_TAIL, "-recip", 0, + "search for recipe", NULL); + psMetadataAddF64(normalizedexpArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background", NAN); + psMetadataAddF64(normalizedexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev", NAN); + psMetadataAddF64(normalizedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddStr(normalizedexpArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + psMetadataAddU64(normalizedexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(normalizedexpArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(normalizedexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -toresidimfile + psMetadata *toresidimfileArgs = psMetadataAlloc(); + psMetadataAddU64(toresidimfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(toresidimfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -normalizedimfile + psMetadata *normalizedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(normalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID", NULL); + psMetadataAddS32(normalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, + "search for iteration number", 0); + psMetadataAddStr(normalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search for class ID", NULL); + psMetadataAddStr(normalizedimfileArgs, PS_LIST_TAIL, "-recip", 0, + "search for recipe", NULL); + psMetadataAddU64(normalizedimfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(normalizedimfileArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(normalizedimfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -toresidexp + psMetadata *toresidexpArgs = psMetadataAlloc(); + psMetadataAddU64(toresidexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(toresidexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addresidimfile + psMetadata *addresidimfileArgs = psMetadataAlloc(); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddS32(addresidimfileArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-exp_tag", 0, + "define detrend ID (required)", NULL); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "define class ID (required)", NULL); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-uri", 0, + "define resid file URI (required)", NULL); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-recip", 0, + "define recipe (required)", NULL); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background (required)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev (required)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-bin_stdev", 0, + "define exposure background binned stdev", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_0", 0, + "define fringe slope (0th term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_1", 0, + "define fringe slope (1st term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-fringe_2", 0, + "define fringe slope (2nd term)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_1", 0, + "define user statistic (1)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_2", 0, + "define user statistic (2)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_3", 0, + "define user statistic (3)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_4", 0, + "define user statistic (4)", NAN); + psMetadataAddF64(addresidimfileArgs, PS_LIST_TAIL, "-user_5", 0, + "define user statistic (5)", NAN); + psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + psMetadataAddS16(addresidimfileArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + + // -residimfile + psMetadata *residimfileArgs = psMetadataAlloc(); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID", NULL); + psMetadataAddS32(residimfileArgs, PS_LIST_TAIL, "-iteration", 0, + "search for iteration number", 0); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-exp_tag", 0, + "define detrend ID", NULL); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search for class ID", NULL); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-recip", 0, + "search for recipe", NULL); + psMetadataAddU64(residimfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addresidexp + psMetadata *addresidexpArgs = psMetadataAlloc(); + psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddS32(addresidexpArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-exp_tag", 0, + "define detrend ID (required)", NULL); + psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-recip", 0, + "define recipe (required)", NULL); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background (required)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev (required)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bin_stdev", 0, + "define exposure background binned stdev", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_0", 0, + "define fringe slope (0th term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_1", 0, + "define fringe slope (1st term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-fringe_2", 0, + "define fringe slope (2nd term)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_1", 0, + "define user statistic (1)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_2", 0, + "define user statistic (2)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_3", 0, + "define user statistic (3)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_4", 0, + "define user statistic (4)", NAN); + psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-user_5", 0, + "define user statistic (5)", NAN); + psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + psMetadataAddS16(addresidexpArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + psMetadataAddBool(addresidexpArgs, PS_LIST_TAIL, "-reject", 0, + "exposure is not to be stacked in the next iteration", false); + + // -residexp + psMetadata *residexpArgs = psMetadataAlloc(); + psMetadataAddStr(residexpArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID", NULL); + psMetadataAddS32(residexpArgs, PS_LIST_TAIL, "-iteration", 0, + "search for iteration number", 0); + psMetadataAddStr(residexpArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search for exp ID", NULL); + psMetadataAddStr(residexpArgs, PS_LIST_TAIL, "-recip", 0, + "search for recipe", NULL); + psMetadataAddU64(residexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(residexpArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(residexpArgs, PS_LIST_TAIL, "-reject", 0, + "search for acceptable residuals", false); + psMetadataAddBool(residexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -residdetrun + psMetadata *residdetrunArgs = psMetadataAlloc(); + psMetadataAddBool(residdetrunArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + psMetadataAddU64(residdetrunArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + + // -updateresidexp + psMetadata *updateresidexpArgs = psMetadataAlloc(); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID", NULL); + psMetadataAddS32(updateresidexpArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-exp_tag", 0, + "define exp ID", NULL); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-recip", 0, + "define recipe", NULL); + psMetadataAddF64(updateresidexpArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background", NAN); + psMetadataAddF64(updateresidexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev", NAN); + psMetadataAddF64(updateresidexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + psMetadataAddBool(updateresidexpArgs, PS_LIST_TAIL, "-reject", 0, + "exposure is not to be stacked in the next iteration", false); + + // -adddetrunsummary + psMetadata *adddetrunsummaryArgs = psMetadataAlloc(); + psMetadataAddStr(adddetrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddS32(adddetrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, + "define iteration number", 0); + psMetadataAddF64(adddetrunsummaryArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background (required)", NAN); + psMetadataAddF64(adddetrunsummaryArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev (required)", NAN); + psMetadataAddF64(adddetrunsummaryArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddS16(adddetrunsummaryArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + psMetadataAddBool(adddetrunsummaryArgs, PS_LIST_TAIL, "-accept", 0, + "declare that this detrun iteration is accepted as a master", false); + + // -detrunsummary + psMetadata *detrunsummaryArgs = psMetadataAlloc(); + psMetadataAddStr(detrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend ID", NULL); + psMetadataAddS32(detrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, + "search for iteration number", 0); + psMetadataAddBool(detrunsummaryArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(detrunsummaryArgs, PS_LIST_TAIL, "-reject", 0, + "search for acceptable residuals", false); + psMetadataAddBool(detrunsummaryArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -updatedetrun + psMetadata *updatedetrunArgs = psMetadataAlloc(); + psMetadataAddStr(updatedetrunArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend master for detrend ID (required)", NULL); + psMetadataAddBool(updatedetrunArgs, PS_LIST_TAIL, "-again", 0, + "start a new iteration of this detrend run", false); + psMetadataAddStr(updatedetrunArgs, PS_LIST_TAIL, "-state", 0, + "set the state of this detrend run", false); + + // -rerun + psMetadata *rerunArgs = psMetadataAlloc(); + psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-det_id", 0, + "search for detrend master for detrend ID (required)", NULL); + psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-exp_tag", PS_META_DUPLICATE_OK, + "include this exposure (multiple OK, required)", NULL); + + // -register_detrend + psMetadata *register_detrendArgs = psMetadataAlloc(); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-det_type", 0, + "define the type of detrend run (required)", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-mode", 0, + "define the mode of this detrend run", "master"); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-filelevel", 0, + "define filelevel (required)", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-workdir", 0, + "define workdir (required)", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-inst", 0, + "define camera", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-telescope", 0, + "define telescope", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-exp_type", 0, + "define exposure type", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-filter", 0, + "define filter ", NULL); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-airmass_min", 0, + "define min airmass", NAN); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-airmass_max", 0, + "define max airmass", NAN); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-exp_time_min", 0, + "define min exposure time", NAN); + psMetadataAddF32(register_detrendArgs, PS_LIST_TAIL, "-exp_time_max", 0, + "define max exposure time", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, + "define min ccd tempature", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, + "define max ccd tempature", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-posang_min", 0, + "define min rotator position angle", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-posang_max", 0, + "define max rotator position angle", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-solang_min", 0, + "define min solar angle", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-solang_max", 0, + "define max solar angle", NAN); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-registered", 0, + "time detrend run was registered", now); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-time_begin", 0, + "detrend applyes to exposures taken during this peroid", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-time_end", 0, + "detrend applyes to exposures taken during this peroid", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-use_begin", 0, + "start of detrend run applicable peroid", NULL); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-use_end", 0, + "end of detrend run applicable peroid", NULL); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddStr(register_detrendArgs, PS_LIST_TAIL, "-parent", 0, + "define parent det_id", NULL); + psMetadataAddBool(register_detrendArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -register_detrend_imfile + psMetadata *register_detrend_imfileArgs = psMetadataAlloc(); + psMetadataAddStr(register_detrend_imfileArgs, PS_LIST_TAIL, "-det_id", 0, + "define detrend ID (required)", NULL); + psMetadataAddStr(register_detrend_imfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search for class ID (required)", NULL); + psMetadataAddStr(register_detrend_imfileArgs, PS_LIST_TAIL, "-uri", 0, + "define resid file URI (required)", NULL); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev", NAN); + psMetadataAddF64(register_detrend_imfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposure background mean stdev", NAN); + psMetadataAddStr(register_detrend_imfileArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location", NULL); + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXTOOL_ADD_MODE("-pending", "list active detruns", DETTOOL_MODE_PENDING, pendingArgs); + PXTOOL_ADD_MODE("-definebytag", "", DETTOOL_MODE_DEFINEBYTAG, definebytagArgs); + PXTOOL_ADD_MODE("-definebyquery", "", DETTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXTOOL_ADD_MODE("-definebydetrun", "", DETTOOL_MODE_DEFINEBYDETRUN, definebydetrunArgs); + PXTOOL_ADD_MODE("-raw", "", DETTOOL_MODE_RAW, rawArgs); + PXTOOL_ADD_MODE("-runs", "", DETTOOL_MODE_RUNS, runsArgs); + PXTOOL_ADD_MODE("-childlessrun", "", DETTOOL_MODE_CHILDLESSRUN, childlessrunArgs); + PXTOOL_ADD_MODE("-input", "", DETTOOL_MODE_INPUT, inputArgs); + PXTOOL_ADD_MODE("-toprocessedimfile", "", DETTOOL_MODE_TOPROCESSEDIMFILE, toprocessedimfileArgs); + PXTOOL_ADD_MODE("-addprocessedimfile", "", DETTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); + PXTOOL_ADD_MODE("-processedimfile", "", DETTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); + PXTOOL_ADD_MODE("-toprocessedexp", "", DETTOOL_MODE_TOPROCESSEDEXP, toprocessedexpArgs); + PXTOOL_ADD_MODE("-addprocessedexp", "", DETTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs); + PXTOOL_ADD_MODE("-processedexp", "", DETTOOL_MODE_PROCESSEDEXP, processedexpArgs); + PXTOOL_ADD_MODE("-tostacked", "", DETTOOL_MODE_TOSTACKED, tostackedArgs); + PXTOOL_ADD_MODE("-addstacked", "", DETTOOL_MODE_ADDSTACKED, addstackedArgs); + PXTOOL_ADD_MODE("-stacked", "", DETTOOL_MODE_STACKED, stackedArgs); + PXTOOL_ADD_MODE("-tonormalize", "", DETTOOL_MODE_TONORMALIZE, tonormalizeArgs); + PXTOOL_ADD_MODE("-addnormalizedstat", "", DETTOOL_MODE_ADDNORMALIZEDSTAT, addnormstatArgs); + PXTOOL_ADD_MODE("-tonormalizedstat", "", DETTOOL_MODE_TONORMALIZEDSTAT, tonormstatArgs); + PXTOOL_ADD_MODE("-addnormalizedimfile", "", DETTOOL_MODE_ADDNORMALIZEDIMFILE,addnormalizedimfileArgs); + PXTOOL_ADD_MODE("-normalizedimfile","", DETTOOL_MODE_NORMALIZEDIMFILE, normalizedimfileArgs); + PXTOOL_ADD_MODE("-tonormalizedexp", "", DETTOOL_MODE_TONORMALIZEDEXP, tonormalizedexpArgs); + PXTOOL_ADD_MODE("-addnormalizedexp", "", DETTOOL_MODE_ADDNORMALIZEDEXP, addnormalizedexpArgs); + PXTOOL_ADD_MODE("-normalizedexp", "", DETTOOL_MODE_NORMALIZEDEXP, normalizedexpArgs); + PXTOOL_ADD_MODE("-toresidimfile", "", DETTOOL_MODE_TORESIDIMFILE, toresidimfileArgs); + PXTOOL_ADD_MODE("-addresidimfile", "", DETTOOL_MODE_ADDRESIDIMFILE, addresidimfileArgs); + PXTOOL_ADD_MODE("-residimfile", "", DETTOOL_MODE_RESIDIMFILE, residimfileArgs); + PXTOOL_ADD_MODE("-toresidexp", "", DETTOOL_MODE_TORESIDEXP, toresidexpArgs); + PXTOOL_ADD_MODE("-addresidexp", "", DETTOOL_MODE_ADDRESIDEXP, addresidexpArgs); + PXTOOL_ADD_MODE("-residexp", "", DETTOOL_MODE_RESIDEXP, residexpArgs); + PXTOOL_ADD_MODE("-residdetrun", "", DETTOOL_MODE_RESIDDETRUN, residdetrunArgs); + PXTOOL_ADD_MODE("-updateresidexp", "", DETTOOL_MODE_UPDATERESIDEXP,updateresidexpArgs); + PXTOOL_ADD_MODE("-adddetrunsummary", "", DETTOOL_MODE_ADDDETRUNSUMMARY,adddetrunsummaryArgs); + PXTOOL_ADD_MODE("-detrunsummary", "", DETTOOL_MODE_DETRUNSUMMARY,detrunsummaryArgs); + PXTOOL_ADD_MODE("-updatedetrun", "", DETTOOL_MODE_UPDATEDETRUN, updatedetrunArgs); + PXTOOL_ADD_MODE("-rerun", "", DETTOOL_MODE_RERUN, rerunArgs); + PXTOOL_ADD_MODE("-register_detrend", "", DETTOOL_MODE_REGISTER_DETREND, register_detrendArgs); + PXTOOL_ADD_MODE("-register_detrend_imfile", "", DETTOOL_MODE_REGISTER_DETREND_IMFILE, register_detrend_imfileArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // setup search criterion +#define addWhereStr(name) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, #name, 0, "==", str)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + + // generate SQL where clause + config->where = psMetadataAlloc(); + + addWhereStr(det_id); + { + int n = 0; + bool status = false; + if ((n = psMetadataLookupS32(&status, config->args, "-iteration"))) { + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, "iteration", 0, "==", n)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(config); + return NULL; + } + } + } + addWhereStr(det_type); + addWhereStr(exp_tag); + addWhereStr(class_id); + // convert '-inst' to 'camera' + { + psString str = NULL; + bool status = false; + if ((str = psMetadataLookupStr(&status, config->args, "-inst"))) { + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, "camera", 0, "==", str)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); + psFree(config); + return NULL; + } + } + } + addWhereStr(telescope); + addWhereStr(exp_type); + { + int n = 0; + bool status = false; + if ((n = psMetadataLookupS32(&status, config->args, "-imfiles"))) { + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, "imfiles", 0, "==", n)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item imfiles"); + psFree(config); + return NULL; + } + } + } + addWhereStr(filter); + addWhereStr(recipe); + { + int n = 0; + bool status = false; + if ((n = psMetadataLookupS32(&status, config->args, "-guide_version"))) { + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, "guide_version", 0, "==", n)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item guide_version"); + psFree(config); + return NULL; + } + } + } + { + bool boolean = false; + bool status = false; + + // map reject -> !accept + if ((boolean = psMetadataLookupBool(&status, config->args, "-reject"))) { + if (!psMetadataAddBool(config->where, PS_LIST_TAIL, "accept", 0, "==", !boolean)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item reject"); + psFree(config); + return NULL; + } + } + } + + if (config->where->list->n < 1) { + psFree(config->where); + config->where = NULL; + } + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = pmConfigDB(config->modules); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/jh_branches/backtrack/ippTools/src/difftool.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/difftool.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/difftool.c (revision 22250) @@ -0,0 +1,672 @@ +/* + * difftool.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "difftool.h" + +static bool definerunMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool addinputskyfileMode(pxConfig *config); +static bool inputskyfileMode(pxConfig *config); +static bool todiffskyfileMode(pxConfig *config); +static bool adddiffskyfileMode(pxConfig *config); +static bool diffskyfileMode(pxConfig *config); + +static bool setdiffRunState(pxConfig *config, const char *warp_id, const char *state); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = difftoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(DIFFTOOL_MODE_DEFINERUN, definerunMode); + MODECASE(DIFFTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(DIFFTOOL_MODE_ADDINPUTSKYFILE, addinputskyfileMode); + MODECASE(DIFFTOOL_MODE_INPUTSKYFILE, inputskyfileMode); + MODECASE(DIFFTOOL_MODE_TODIFFSKYFILE, todiffskyfileMode); + MODECASE(DIFFTOOL_MODE_ADDDIFFSKYFILE, adddiffskyfileMode); + MODECASE(DIFFTOOL_MODE_DIFFSKYFILE, diffskyfileMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + bool status = false; + psString workdir = psMetadataLookupStr(&status, config->args, "-workdir"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -workdir"); + return false; + } + if (!workdir) { + psError(PS_ERR_UNKNOWN, true, "-workdir is required"); + return false; + } + + psString skycell_id = psMetadataLookupStr(&status, config->args, "-skycell_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -skycell_id"); + return false; + } + if (!skycell_id) { + psError(PS_ERR_UNKNOWN, true, "-skycell_id is required"); + return false; + } + + psString tess_id = psMetadataLookupStr(&status, config->args, "-tess_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -tess_id"); + return false; + } + if (!tess_id) { + psError(PS_ERR_UNKNOWN, true, "-tess_id is required"); + return false; + } + + psTime *registered = NULL; + { + psString registeredStr = psMetadataLookupStr(&status, config->args, "-registered"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -registered"); + return false; + } + // pass through NULL as this is an optional field + if (registeredStr) { + registered = psTimeFromISO(registeredStr, PS_TIME_UTC); + } else { + registered = NULL; + } + } + + diffRunRow *run = diffRunRowAlloc( + 0, // ID + "reg", // state + workdir, + NULL, // dvodb + registered, + skycell_id, + tess_id + ); + psFree(registered); + if (!run) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc diffRun object"); + return true; + } + if (!diffRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return true; + } + + // get the assigned diff_id + run->diff_id = psDBLastInsertID(config->dbh); + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(run); + return false; + } + } + + if (!diffRunPrintObject(stdout, run, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(run); + return false; + } + + psFree(run); + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString diff_id = psMetadataLookupStr(&status, config->args, "-diff_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -diff_id"); + return false; + } + if (!diff_id) { + psError(PS_ERR_UNKNOWN, true, "-diff_id is required"); + return false; + } + + psString state = psMetadataLookupStr(&status, config->args, "-state"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -state"); + return false; + } + if (!state) { + psError(PS_ERR_UNKNOWN, true, "-state is required"); + return false; + } + + if (state) { + // set detRun.state to state + return setdiffRunState(config, diff_id, state); + } + + return true; +} + + +static bool addinputskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString diff_id = psMetadataLookupStr(&status, config->args, "-diff_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -diff_id"); + return false; + } + if (!diff_id) { + psError(PS_ERR_UNKNOWN, true, "-diff_id is required"); + return false; + } + + psString warp_id = psMetadataLookupStr(&status, config->args, "-warp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -warp_id"); + return false; + } + if (!diff_id) { + psError(PS_ERR_UNKNOWN, true, "-diff_id is required"); + return false; + } + + psString skycell_id = psMetadataLookupStr(&status, config->args, "-skycell_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -skycell_id"); + return false; + } + if (!skycell_id) { + psError(PS_ERR_UNKNOWN, true, "-skycell_id is required"); + return false; + } + + psString tess_id = psMetadataLookupStr(&status, config->args, "-tess_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -tess_id"); + return false; + } + if (!tess_id) { + psError(PS_ERR_UNKNOWN, true, "-tess_id is required"); + return false; + } + + psString kind = psMetadataLookupStr(&status, config->args, "-kind"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -kind"); + return false; + } + + // defaults to false + bool template = psMetadataLookupBool(&status, config->args, "-template"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -template"); + return false; + } + + // XXX need to validate the warp_id here + // XXX instead of validiting it here we should just use forgein key + // constrants + if (!diffInputSkyfileInsert(config->dbh, + (psS64)atoll(diff_id), + (psS64)atoll(warp_id), + skycell_id, + tess_id, + kind, + template + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool inputskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit"); + return false; + } + + // find all rawImfiles matching the default query + psString query = pxDataGet("difftool_inputskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "diffInputSkyfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (psArrayLength(output)) { + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffInputSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool todiffskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("difftool_todiffskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "diffSkyfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (psArrayLength(output)) { + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool adddiffskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString diff_id = psMetadataLookupStr(&status, config->args, "-diff_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -diff_id"); + return false; + } + if (!diff_id) { + psError(PS_ERR_UNKNOWN, true, "-diff_id is required"); + return false; + } + + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + if (!path_base) { + psError(PS_ERR_UNKNOWN, true, "-path_base is required"); + return false; + } + + // optional + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!diffSkyfileInsert(config->dbh, + (psS64)atoll(diff_id), + uri, + path_base, + bg, + bg_stdev + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!setdiffRunState(config, diff_id, "stop")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool diffskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("difftool_skyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "diffSkyfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (psArrayLength(output)) { + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool setdiffRunState(pxConfig *config, const char *diff_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(diff_id, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "reg", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid diffRun state: %s", state); + return false; + } + + char *query = "UPDATE diffRun SET state = '%s' WHERE diff_id = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, state, diff_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for diff_id %s", diff_id); + return false; + } + + return true; +} Index: /branches/jh_branches/backtrack/ippTools/src/difftool.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/difftool.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/difftool.h (revision 22250) @@ -0,0 +1,38 @@ +/* + * difftool.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef DIFFTOOL_H +#define DIFFTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + DIFFTOOL_MODE_NONE = 0x0, + DIFFTOOL_MODE_DEFINERUN, + DIFFTOOL_MODE_UPDATERUN, + DIFFTOOL_MODE_ADDINPUTSKYFILE, + DIFFTOOL_MODE_INPUTSKYFILE, + DIFFTOOL_MODE_TODIFFSKYFILE, + DIFFTOOL_MODE_ADDDIFFSKYFILE, + DIFFTOOL_MODE_DIFFSKYFILE, +} difftoolMode; + +pxConfig *difftoolConfig(pxConfig *config, int argc, char **argv); + +#endif // DIFFTOOL_H Index: /branches/jh_branches/backtrack/ippTools/src/difftoolConfig.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/difftoolConfig.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/difftoolConfig.c (revision 22250) @@ -0,0 +1,240 @@ +/* + * difftoolConfig.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "difftool.h" + +pxConfig *difftoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *time = psTimeGetNow(PS_TIME_TAI); + psString now = psTimeToISO(time); + psFree(time); + + // -definerun + psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, + "define workdir (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-registered", 0, + "time detrend run was registered", now); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-skycell_id", 0, + "define skycell ID (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, + "define tessellation ID (required)", NULL); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-diff_id", 0, + "define diff ID (required)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, + "set state (required)", NULL); +#if 0 + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-workdir", 0, + "define workdir (required)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-registered", 0, + "time detrend run was registered", now); +#endif + + // -addinputskyfile + psMetadata *addinputskyfileArgs = psMetadataAlloc(); + psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, + "define diff ID (required)", NULL); + psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, + "define warp ID (required)", NULL); + psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, + "define by skycell ID (required)", NULL); + psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, + "dfine tess ID (required)", NULL); + psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-kind", 0, + "define kind", NULL); + psMetadataAddBool(addinputskyfileArgs, PS_LIST_TAIL, "-template", 0, + "this sky cell file is the subtrahend", false); + + // -inputskyfile + psMetadata *inputskyfileArgs = psMetadataAlloc(); + psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, + "search by diff ID", NULL); + psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, + "search by warp ID", NULL); + psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, + "search by skycell ID", NULL); + psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, + "search by tess ID", NULL); + psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-kind", 0, + "search by kind", NULL); + psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-template", 0, + "search by subtrahend", false); + psMetadataAddU64(inputskyfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -todiffskyfile + psMetadata *todiffskyfileArgs = psMetadataAlloc(); + psMetadataAddStr(todiffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, + "search by diff ID", NULL); + psMetadataAddU64(todiffskyfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(todiffskyfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -adddiffskyfile + psMetadata *adddiffskyfileArgs = psMetadataAlloc(); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, + "define warp ID (required)", NULL); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-uri", 0, + "define URI of file (required)", 0); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location (required)", 0); + psMetadataAddF64(adddiffskyfileArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background", NAN); + psMetadataAddF64(adddiffskyfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background mean stdev", NAN); + + // -diffskyfile + psMetadata *diffskyfileArgs = psMetadataAlloc(); + psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, + "search by warp ID", NULL); + psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-skycell_id", 0, + "define skycell ID", NULL); + psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, + "define tessellation ID", NULL); + psMetadataAddU64(diffskyfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXTOOL_ADD_MODE("-definerun", "", DIFFTOOL_MODE_DEFINERUN, definerunArgs); + PXTOOL_ADD_MODE("-updaterun", "", DIFFTOOL_MODE_UPDATERUN, updaterunArgs); + PXTOOL_ADD_MODE("-addinputskyfile", "", DIFFTOOL_MODE_ADDINPUTSKYFILE, addinputskyfileArgs); + PXTOOL_ADD_MODE("-inputskyfile", "", DIFFTOOL_MODE_INPUTSKYFILE, inputskyfileArgs); + PXTOOL_ADD_MODE("-todiffskyfile", "", DIFFTOOL_MODE_TODIFFSKYFILE, todiffskyfileArgs); + PXTOOL_ADD_MODE("-adddiffskyfile", "", DIFFTOOL_MODE_ADDDIFFSKYFILE, adddiffskyfileArgs); + PXTOOL_ADD_MODE("-diffskyfile", "", DIFFTOOL_MODE_DIFFSKYFILE, diffskyfileArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // setup search criterion +#define addWhereStr(name) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, #name, 0, "==", str)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define addWhereS32(name) \ +{ \ + psS32 s32 = 0; \ + bool status = false; \ + if ((s32= psMetadataLookupS32(&status, config->args, "-" #name))) { \ + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, #name, 0, "==", s32)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + + + // generate SQL where clause + config->where = psMetadataAlloc(); + +{ + psString str = NULL; + bool status = false; + if ((str = psMetadataLookupStr(&status, config->args, "-diff_id"))) { + if (!psMetadataAddS64(config->where, PS_LIST_TAIL, "diff_id", 0, "==", (psS64)atoll(str))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item diff_id"); + psFree(config); + return NULL; + } + } +} +{ + psString str = NULL; + bool status = false; + if ((str = psMetadataLookupStr(&status, config->args, "-warp_id"))) { + if (!psMetadataAddS64(config->where, PS_LIST_TAIL, "warp_id", 0, "==", (psS64)atoll(str))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item warp_id"); + psFree(config); + return NULL; + } + } +} + addWhereStr(skycell_id); + addWhereStr(tess_id); + + if (config->where->list->n < 1) { + psFree(config->where); + config->where = NULL; + } + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = pmConfigDB(config->modules); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/jh_branches/backtrack/ippTools/src/guidetool.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/guidetool.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/guidetool.c (revision 22250) @@ -0,0 +1,201 @@ +/* + * guidetool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "guidetool.h" + +static bool pendingMode(pxConfig *config); +static bool defineMode(pxConfig *config); +static guidePendingExpRow *rawScienceToguidePendingExp(pxConfig *config, rawScienceExpRow *exp); + +int main(int argc, char **argv) +{ + pxConfig *config = guidetoolConfig(NULL, argc, argv); + + switch (config->mode) { + case PX_MODE_PENDING: + if (!pendingMode(config)) { + goto FAIL; + } + break; + case PX_MODE_DEFINE: + if (!defineMode(config)) { + goto FAIL; + } + break; + default: + psAbort("invalid option (this should not happen)"); + } + + exit(EXIT_SUCCESS); + +FAIL: + psFree(config); + exit(EXIT_FAILURE); +} + +static bool pendingMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // return all guidePendingExp rows unless there CLI search options + psArray *guidePendingExp = guidePendingExpSelectRowObjects( + config->dbh, + config->where ? config->where : NULL, + 0 + ); + if (!guidePendingExp) { + psError(PS_ERR_UNKNOWN, false, "no guidePendingExp found"); + return false; + } + + psMetadata *output = psMetadataAlloc(); + + for (long i = 0; i < psArrayLength(guidePendingExp); i++) { + psMetadata *md = guidePendingExpMetadataFromObject(guidePendingExp->data[i]); + psMetadataAddMetadata(output, PS_LIST_TAIL, "guidePendingExp", + PS_META_DUPLICATE_OK, NULL, md); + psFree(md); + } + + psString str = psMetadataConfigFormat(output); + psFree(output); + + fprintf(stdout, "%s\n", str); + psFree(str); + + return true; +} + +static bool defineMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // return all rawScienceExp rows unless there CLI search options + psArray *rawScienceExps = rawScienceExpSelectRowObjects( + config->dbh, + config->where ? config->where : NULL, + 0 + ); + + if (!rawScienceExps) { + psError(PS_ERR_UNKNOWN, false, "no rawScienceExp rows found"); + return true; + } + + // insert the rawScienceExps into guidePendingExp + // XXX for the being we're going to skip checking for duplicate entries + // XXX this will have to implimented unless duplicate key insertion + // XXX support is implimented in psDB + for (long i = 0; i < psArrayLength(rawScienceExps); i++) { + guidePendingExpRow * new = rawScienceToguidePendingExp( + config, rawScienceExps->data[i]); + if (!guidePendingExpInsertObject(config->dbh, new)) { + psError(PS_ERR_UNKNOWN, false, "dbh access failed"); + return false; + } + psFree(new); + } + + return true; +} + +static guidePendingExpRow *rawScienceToguidePendingExp(pxConfig *config, rawScienceExpRow *exp) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(exp, NULL); + + bool status = false; + psString filter = psMetadataLookupStr(&status, config->args, "-filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter"); + return false; + } + if (!filter) { + psError(PS_ERR_UNKNOWN, true, "-filter is required"); + return false; + } + psF32 airmass = psMetadataLookupF32(&status, config->args, "-airmass"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass"); + return false; + } + if (!airmass) { + psError(PS_ERR_UNKNOWN, true, "-airmass is required"); + return false; + } + psF64 ra = psMetadataLookupF64(&status, config->args, "-ra"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ra"); + return false; + } + if (!ra) { + psError(PS_ERR_UNKNOWN, true, "-airmass is required"); + return false; + } + psF64 dec = psMetadataLookupF64(&status, config->args, "-dec"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -dec"); + return false; + } + if (!dec) { + psError(PS_ERR_UNKNOWN, true, "-dec is required"); + return false; + } + psF32 exp_time = psMetadataLookupF32(&status, config->args, "-exp_time"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time"); + return false; + } + if (!exp_time) { + psError(PS_ERR_UNKNOWN, true, "-exp_time is required"); + return false; + } + psF64 background = psMetadataLookupF64(&status, config->args, "-background"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -background"); + return false; + } + if (!exp_time) { + psError(PS_ERR_UNKNOWN, true, "-background is required"); + return false; + } + + return guidePendingExpRowAlloc( + exp->exp_tag, + exp->camera, + exp->telescope, + exp->exp_type, + exp->imfiles, + filter, + airmass, + ra, + dec, + exp_time, + background, + "my recipe", + 0xff // XXX calc version number + ); +} Index: /branches/jh_branches/backtrack/ippTools/src/guidetool.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/guidetool.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/guidetool.h (revision 22250) @@ -0,0 +1,27 @@ +/* + * guidetool.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef GUIDETOOL_H +#define GUIDETOOL_H 1 + +#include "pxtools.h" + +pxConfig *guidetoolConfig(pxConfig *config, int argc, char **argv); + +#endif // GUIDETOOL_H Index: /branches/jh_branches/backtrack/ippTools/src/guidetoolConfig.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/guidetoolConfig.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/guidetoolConfig.c (revision 22250) @@ -0,0 +1,205 @@ +/* + * guidetoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" + +// this function can not fail -- exits on error + +pxConfig *guidetoolConfig(pxConfig *config, int argc, char **argv) { + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + exit(EXIT_FAILURE); + } + + // CLI options + // modes + psMetadata *args = psMetadataAlloc(); + psMetadataAddStr(args , PS_LIST_TAIL, "-pending", 0, + "examine pending image table, write ppImage output", NULL); + psMetadataAddStr(args , PS_LIST_TAIL, "-define", 0, + "update pending image table", NULL); + + // -pending search + psMetadata *pendingArgs = psMetadataAlloc(); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_tag", 0, + "define exposure ID", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst", 0, + "define camera of interest", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope", 0, + "define telescope of interest", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type", 0, + "define exposure type", NULL); + psMetadataAddS32(pendingArgs, PS_LIST_TAIL, "-imfiles", 0, + "define number of imfiles", 0); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter", 0, + "define filter of interest", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-recipe", 0, + "define recipe of interest", NULL); + psMetadataAddS32(pendingArgs, PS_LIST_TAIL, "-guide_version", 0, + "define guide_version of interest", 0); + + // -define inputs + psMetadata *defineArgs = psMetadataAlloc(); + psMetadataAddStr(defineArgs, PS_LIST_TAIL, "-exp_tag", 0, + "define class", NULL); + psMetadataAddStr(defineArgs, PS_LIST_TAIL, "-inst", 0, + "define camera of interest", NULL); + psMetadataAddStr(defineArgs, PS_LIST_TAIL, "-telescope", 0, + "define camera of interest", NULL); + psMetadataAddStr(defineArgs, PS_LIST_TAIL, "-exp_type", 0, + "define class", NULL); + psMetadataAddS32(defineArgs, PS_LIST_TAIL, "-imfiles", 0, + "define number of imfiles", 0); + psMetadataAddStr(defineArgs, PS_LIST_TAIL, "-filter", 0, + "define filter of interest", NULL); + // these two are inputs NOT search parameters + psMetadataAddStr(defineArgs, PS_LIST_TAIL, "-recipe", 0, + "define recipe of interest", NULL); + psMetadataAddS32(defineArgs, PS_LIST_TAIL, "-guide_version", 0, + "define guide_version of interest", 0); + + // find which mode we're running under + int N = 0; + config->mode = PX_MODE_NONE; + if ((N = psArgumentGet (argc, argv, "-pending"))) { + psArgumentRemove (N, &argc, argv); + if (config->mode) { + psError(PS_ERR_UNKNOWN, true, "only one mode selection is allowed"); + } + config->mode = PX_MODE_PENDING; + } + if ((N = psArgumentGet (argc, argv, "-define"))) { + psArgumentRemove (N, &argc, argv); + if (config->mode) { + psError(PS_ERR_UNKNOWN, true, "only one mode selection is allowed"); + } + config->mode = PX_MODE_DEFINE; + } + + // parse CLI for these options + switch (config->mode) { + case PX_MODE_PENDING: + config->args = psMemIncrRefCounter(pendingArgs); + break; + case PX_MODE_DEFINE: + config->args = psMemIncrRefCounter(defineArgs); + break; + default: + config->args = psMemIncrRefCounter(args); + } + + bool argErr = false; + if (config->mode == PX_MODE_NONE) { + argErr = true; + fprintf (stderr, "mode argument is required\n"); + } else if (! psArgumentParse(config->args, &argc, argv) || argc != 1) { + argErr = true; + fprintf (stderr, "error parsing arguments\n"); + } + + if (argErr) { + printf("\nPan-STARRS Guide Star Tool\n"); + printf("Usage: %s []\n\n", argv[0]); + printf(" : -pending | -define\n\n"); + fprintf (stdout, "-pending "); + psArgumentHelp(pendingArgs); + fprintf (stdout, "-define "); + psArgumentHelp(defineArgs); + psFree(args); + psFree(pendingArgs); + psFree(defineArgs); + exit(EXIT_FAILURE); + } + + psFree(args); + psFree(pendingArgs); + psFree(defineArgs); + + // setup search criterion +#define addWhereStr(name) \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, #name, 0, "==", str)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + exit(EXIT_FAILURE); \ + } \ + } + + // generate SQL where clause + config->where = psMetadataAlloc(); + +{ + psString str = NULL; + int n = 0; + bool status = false; + + addWhereStr(exp_tag); + // convert '-inst' to 'camera' + if ((str = psMetadataLookupStr(&status, config->args, "-inst"))) { + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, "camera", 0, "==", str)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); + exit(EXIT_FAILURE); + } + } + addWhereStr(telescope); + addWhereStr(exp_type); + if ((n = psMetadataLookupS32(&status, config->args, "-imfiles"))) { + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, "imfiles", 0, "==", n)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item imfiles"); + exit(EXIT_FAILURE); + } + } + addWhereStr(filter); + addWhereStr(stats); + addWhereStr(recipe); + if ((n = psMetadataLookupS32(&status, config->args, "-guide_version"))) { + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, "guide_version", 0, "==", n)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item guide_version"); + exit(EXIT_FAILURE); + } + } +} + + if (config->where->list->n < 1) { + psFree(config->where); + config->where = NULL; + } + + // define Database handle, if used + config->dbh = pmConfigDB(config->modules); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + exit(EXIT_FAILURE); + } + + return config; +} Index: /branches/jh_branches/backtrack/ippTools/src/pxadmin.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxadmin.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxadmin.c (revision 22250) @@ -0,0 +1,75 @@ +/* + * pxadmin.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pxadmin.h" + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pxAdminConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + case PXADMIN_MODE_RECREATE: + if (!pxDeleteTables(config)) { + goto FAIL; + } + // fall through + case PXADMIN_MODE_CREATE: + if (!pxCreateTables(config)) { + goto FAIL; + } + break; + case PXADMIN_MODE_DELETE: + if (!pxDeleteTables(config)) { + goto FAIL; + } + break; + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} Index: /branches/jh_branches/backtrack/ippTools/src/pxadmin.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxadmin.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxadmin.h (revision 22250) @@ -0,0 +1,34 @@ +/* + * pxadmin.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXADMIN_H +#define PXADMIN_H 1 + +#include "pxtools.h" + +typedef enum { + PXADMIN_MODE_NONE = 0x0, + PXADMIN_MODE_CREATE, + PXADMIN_MODE_DELETE, + PXADMIN_MODE_RECREATE +} pxAdminMode; + +pxConfig *pxAdminConfig (pxConfig *config, int argc, char **argv); + +#endif // PXADMIN_H Index: /branches/jh_branches/backtrack/ippTools/src/pxadminConfig.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxadminConfig.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxadminConfig.c (revision 22250) @@ -0,0 +1,119 @@ +/* + * pxadminConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pxadmin.h" + +pxConfig *pxAdminConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // Parse other command-line arguments + psMetadata *arguments = psMetadataAlloc(); // The arguments, with default values + + int N; + config->mode = PXADMIN_MODE_NONE; + if ((N = psArgumentGet(argc, argv, "-create"))) { + psArgumentRemove(N, &argc, argv); + if (config->mode) { + psAbort("only one mode selection is allowed"); + } + config->mode = PXADMIN_MODE_CREATE; + } + if ((N = psArgumentGet(argc, argv, "-delete"))) { + psArgumentRemove(N, &argc, argv); + if (config->mode) { + psAbort("only one mode selection is allowed"); + } + config->mode = PXADMIN_MODE_DELETE; + } + if ((N = psArgumentGet(argc, argv, "-recreate"))) { + psArgumentRemove(N, &argc, argv); + if (config->mode) { + psAbort("only one mode selection is allowed"); + } + config->mode = PXADMIN_MODE_RECREATE; + } + + // paul's argument parsing convention requires: -key value + psMetadataAddStr(arguments, PS_LIST_TAIL, "-create", 0, + "create the P2 tables", ""); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-delete", 0, + "delete the P2 tables", ""); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-recreate", 0, + "delete and recreate the P2 tables", ""); + + if (config->mode == PXADMIN_MODE_NONE) { + fprintf (stderr, "admin mode not specified\n"); + psArgumentHelp(arguments); + psFree(arguments); + psFree(config); + return NULL; + } + + if ((N = psArgumentGet (argc, argv, "-help"))) { + psArgumentHelp(arguments); + psFree(arguments); + psFree(config); + return NULL; + } + + if (! psArgumentParse(arguments, &argc, argv) || argc != 1) { + printf("\nPan-STARRS Phase 2 Admin Tool\n\n"); + printf("Usage: %s [mode]\n", argv[0]); + printf(" [mode] : -create | -delete\n\n"); + psArgumentHelp(arguments); + psFree(arguments); + psFree(config); + return NULL; + } + + psFree(arguments); + + // define Database handle, if used + config->dbh = pmConfigDB(config->modules); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + // save argv/argc + config->argv = argv; + config->argc = argc; + + return config; +} Index: /branches/jh_branches/backtrack/ippTools/src/pxcam.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxcam.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxcam.c (revision 22250) @@ -0,0 +1,104 @@ +/* + * pxcam.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxcam.h" + +bool pxcamRunSetState(pxConfig *config, psS64 cam_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "reg", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid camRun state: %s", state); + return false; + } + + char *query = "UPDATE camRun SET state = '%s' WHERE cam_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, cam_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for cam_id %" PRId64, cam_id); + return false; + } + + return true; +} + + +bool pxcamQueueByChipID(pxConfig *config, + psS64 chip_id, + psString workdir, + psString label, + psString recipe, + psString expgroup, + psString dvodb) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_tags from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("camtool_queue_chip_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + } + + // queue the exp + // XXX chip_id is being cast here work around psS64 have a different type + // different on 32/64 + if (!p_psDBRunQuery(config->dbh, query, + "run", // state + workdir ? workdir : "NULL", + "dirty", //workdir_state + label ? label : "NULL", + recipe ? recipe : "NULL", + expgroup ? expgroup : "NULL", + dvodb ? dvodb : "NULL", + (long long)chip_id + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row"); + return false; + } + + return true; +} Index: /branches/jh_branches/backtrack/ippTools/src/pxcam.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxcam.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxcam.h (revision 22250) @@ -0,0 +1,37 @@ +/* + * pxcam.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXCAM_H +#define PXCAM_H 1 + +#include + +#include "pxtools.h" + +bool pxcamRunSetState(pxConfig *config, psS64 cam_id, const char *state); + +bool pxcamQueueByChipID(pxConfig *config, + psS64 chip_id, + psString workdir, + psString label, + psString recipe, + psString expgroup, + psString dvodb); + +#endif // PXCAM_H Index: /branches/jh_branches/backtrack/ippTools/src/pxchip.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxchip.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxchip.c (revision 22250) @@ -0,0 +1,126 @@ +/* + * pxchip.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxchip.h" + +bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "reg", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid chipRun state: %s", state); + return false; + } + + char *query = "UPDATE chipRun SET state = '%s' WHERE chip_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query, state, chip_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for chip_id %" PRId64, chip_id); + return false; + } + + return true; +} + + +bool pxchipQueueByExpTag(pxConfig *config, + psString exp_tag, + psString workdir, + psString label, + psString reduction, + psString expgroup, + psString dvodb) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // start a transaction + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // create a chipRun + if (!chipRunInsert(config->dbh, + 0x0, + "run", // state + workdir, + "dirty", // workdir_state + label, + reduction, + expgroup, + dvodb) + ) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + +#if 0 + // get the chip_id + long chip_id = psDBLastInsertID(config->dbh); +#endif + + // populate the chipInputImfile table from rawImfile based on exp_tag, + // where the coresponding newExp.state entry is set to stop + psString query = pxDataGetCached("chiptool_queuerawimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // queue the imfiles for the exp_tag + if (!p_psDBRunQuery(config->dbh, query, exp_tag)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row"); + return false; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} Index: /branches/jh_branches/backtrack/ippTools/src/pxchip.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxchip.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxchip.h (revision 22250) @@ -0,0 +1,37 @@ +/* + * pxchip.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXCHIP_H +#define PXCHIP_H 1 + +#include + +#include "pxtools.h" + +bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state); + +bool pxchipQueueByExpTag(pxConfig *config, + psString exp_tag, + psString workdir, + psString label, + psString recipe, + psString expgroup, + psString dvodb); + +#endif // PXCHIP_H Index: /branches/jh_branches/backtrack/ippTools/src/pxconfig.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxconfig.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxconfig.c (revision 22250) @@ -0,0 +1,226 @@ +/* + * pxconfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include +#include + +#include "pxtools.h" + +static void pxConfigFree(pxConfig *ptr); + +pxConfig *pxConfigAlloc(void) +{ + pxConfig *config; + + config = psAlloc(sizeof(pxConfig)); + psMemSetDeallocator(config, (psFreeFunc)pxConfigFree); + + config->modeName = NULL; + config->mode = 0; + config->dbh = NULL; + config->modules = NULL; + config->where = NULL; + config->args = NULL; + + return config; +} + +static void pxConfigFree(pxConfig *config) +{ + psFree(config->modeName); + psFree(config->dbh); + psFree(config->modules); + psFree(config->where); + psFree(config->args); +} + +void pxUsage(FILE *stream, int argc, char **argv, const char *modeName, psMetadata *argSet) +{ + fprintf(stream, "Usage: %s %s []\n\n", argv[0], modeName); + fprintf(stream, "%s:\n", modeName); + + psArgumentHelpSimple(stream, argSet); +} + +bool pxGetOptions(FILE *stream, int argc, char **argv, pxConfig *config, psMetadata *modes, psMetadata *argSets) +{ + // figure out what mode we're running in + psMetadataIterator *iter = psMetadataIteratorAlloc(modes, PS_LIST_HEAD, NULL); + psMetadataItem *item = NULL; + while ((item = psMetadataGetAndIncrement(iter))) { + int N = 0; + if ((N = psArgumentGet(argc, argv, item->name))) { + psArgumentRemove(N, &argc, argv); + // check for duplicate mode specification + if (config->mode) { + psError(PS_ERR_UNKNOWN, true, "only one mode selection is allowed"); + fprintf(stream, "only one mode selection is allowed\n"); + pxUsage(stream, argc, argv, "", modes); + psFree(iter); + return NULL; + } + + config->mode = item->data.U32; + config->modeName = psStringCopy(item->name); + + bool status = false; + psMetadata *argset = psMetadataLookupMetadata(&status, argSets, item->name); + // make sure we can find the argSet for the sepcified mode + if (!status) { + psError(PS_ERR_UNKNOWN, true, "can not find arguments for mode"); + fprintf(stderr, "can not find arugments for mode"); + psFree(iter); + return NULL; + } + + config->args = psMemIncrRefCounter(argset); + } + + } + + psFree(iter); + + // make sure we found a mode + if (config->mode == PXTOOL_MODE_NONE) { + psError(PS_ERR_UNKNOWN, true, "mode argument is required"); + fprintf(stderr, "mode argument is required\n"); + pxUsage(stream, argc, argv, "", modes); + return NULL; + } + + // actually parse the command line + if (!psArgumentParse(config->args, &argc, argv)) { + psError(PS_ERR_UNKNOWN, true, "error parsing arguments"); + fprintf(stderr, "error parsing arguments\n"); + pxUsage(stream, argc, argv, config->modeName, config->args); + return NULL; + } + + // look for left overs on the command line + if (argc != 1) { + psError(PS_ERR_UNKNOWN, true, "extra arguments: "); + fprintf(stderr, "extra arguments: "); + for (int i = 1; i < argc; i++) { + fprintf (stderr, "%s ", argv[i]); + } + fprintf(stderr, "\n"); + pxUsage(stream, argc, argv, config->modeName, config->args); + return NULL; + } + + // make sure that all required parameters have been specified + iter = psMetadataIteratorAlloc(config->args, PS_LIST_HEAD, NULL); + item = NULL; // Item from iterator + while ((item = psMetadataGetAndIncrement(iter))) { + if (strstr(item->comment, "require") == NULL) { + continue; + } + if (strstr(item->comment, "(found)") != NULL) { + continue; + } + + switch (item->type) { + case PS_DATA_BOOL: + psError(PS_ERR_UNKNOWN, false, "boolean type can not be required"); + fprintf(stderr, "boolean type can not be required\n"); + psFree(iter); + return NULL; + break; + case PS_DATA_S8: + if (item->data.S8 != INT8_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_S16: + if (item->data.S16 != INT16_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_S32: + if (item->data.S32 != INT32_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_S64: + if (item->data.S64 != INT64_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_U8: + if (item->data.U8 != UINT8_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_U16: + if (item->data.U16 != UINT16_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_U32: + if (item->data.U32 != UINT32_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_U64: + if (item->data.U64 != UINT64_MAX) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_F32: + case PS_DATA_F64: + if (!isnan(item->data.F64)) { + break; + } + goto ARG_REQUIRED; + case PS_DATA_STRING: + case PS_DATA_METADATA: + if (item->data.V) { + break; + } + goto ARG_REQUIRED; + default: + psError(PS_ERR_UNKNOWN, false, "unknown argument type"); + fprintf(stream, "unknown argument type\n"); + psFree(iter); + return NULL; + ARG_REQUIRED: + psError(PS_ERR_UNKNOWN, false, "argument %s is required", item->name); + fprintf(stream, "argument %s is required\n", item->name); + pxUsage(stream, argc, argv, config->modeName, config->args); + psFree(iter); + return NULL; + } + + } + + psFree(iter); + + // save argv/argc + config->argv = argv; + config->argc = argc; + + return true; +} Index: /branches/jh_branches/backtrack/ippTools/src/pxconfig.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxconfig.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxconfig.h (revision 22250) @@ -0,0 +1,39 @@ +/* + * pxconfig.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXCONFIG_H +#define PXCONFIG_H 1 + +#include "pslib.h" +#include "psmodules.h" + +typedef struct { + char *modeName; + int mode; + pmConfig *modules; + psDB *dbh; + psMetadata *args; + psMetadata *where; + int argc; + char **argv; +} pxConfig; + +pxConfig *pxConfigAlloc(void); + +#endif // PXCONFIG_H Index: /branches/jh_branches/backtrack/ippTools/src/pxdata.c.template =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxdata.c.template (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxdata.c.template (revision 22250) @@ -0,0 +1,94 @@ +/* + * pxdata.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include // getenv +#include +#include +#include + +#include + +#include "pxtools.h" +#include "pxdata.h" + +int test_f(const char *path); + +psString pxDataGet(const char *filename) +{ + PS_ASSERT_PTR_NON_NULL(filename, NULL); + + psString filepath = NULL; + + // look to see if PXDATA is set + char *PXDATA = getenv("PXDATA"); + if (PXDATA) { + // if it is, then prepend filename to PXDATA + psStringAppend(&filepath, "%s/%s", PXDATA, filename); + } + + // see if we have a valid filename + if (test_f(filepath)) { + // if we do, slurp the file + psString text = psSlurpFilename(filepath); + if (!text) { + psError(PS_ERR_IO, false, "failed to slurp %s", filepath); + psFree(filepath); + return NULL; + } + psFree(filepath); + + return text; + } + psFree(filepath); + + // look under our share path + psStringAppend(&filepath, "%s/%s", "PKGDATADIR", filename); + + // see if we have a valid filename + if (test_f(filepath)) { + // if we do, slurp the file + psString text = psSlurpFilename(filepath); + if (!text) { + psError(PS_ERR_IO, false, "failed to slurp %s", filepath); + psFree(filepath); + return NULL; + } + psFree(filepath); + + return text; + } + psFree(filepath); + + // couldn't find a matching filename + psError(PS_ERR_IO, true, "can't find a file in search path(s) to match : %s", filename); + + return NULL; +} + +int test_f(const char *path) +{ + // copied from coreutils 6.4 test.c licenced under GPL v2 + struct stat stat_buf; + return (stat (path, &stat_buf) == 0 + && S_ISREG (stat_buf.st_mode)); +} Index: /branches/jh_branches/backtrack/ippTools/src/pxdata.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxdata.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxdata.h (revision 22250) @@ -0,0 +1,28 @@ +/* + * pxdata.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXDATA_H +#define PXDATA_H 1 + +#include + +psString pxDataGet(const char *filename); +psString pxDataGetCached(const char *filename); + +#endif // PXDATA_H Index: /branches/jh_branches/backtrack/ippTools/src/pxerrors.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxerrors.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxerrors.c (revision 22250) @@ -0,0 +1,42 @@ +/* + * pxerrors.c + * + * Copyright (C) 2006 Eugene Magnier + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "pxtools.h" + +psExit pxerrorGetExitStatus () { + + psErrorCode err = psErrorCodeLast (); + switch (err) { + case PXTOOLS_ERR_SYS: + return PS_EXIT_SYS_ERROR; + case PXTOOLS_ERR_CONFIG: + return PS_EXIT_CONFIG_ERROR; + case PXTOOLS_ERR_PROG: + return PS_EXIT_PROG_ERROR; + case PXTOOLS_ERR_DATA: + return PS_EXIT_DATA_ERROR; + default: + return PS_EXIT_UNKNOWN_ERROR; + } + return PS_EXIT_UNKNOWN_ERROR; +} Index: /branches/jh_branches/backtrack/ippTools/src/pxfault.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxfault.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxfault.c (revision 22250) @@ -0,0 +1,65 @@ +/* + * pxfault.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" + +bool pxSetFaultCode(psDB *dbh, const char *tableName, psMetadata *where, psS16 code) +{ + PS_ASSERT_PTR_NON_NULL(dbh, false); + PS_ASSERT_PTR_NON_NULL(tableName, false); + PS_ASSERT_PTR_NON_NULL(where, false); + +#if 0 + // map code string to numeric fault code + psU32 code = mapCodeStrToInt(codeStr); + if (code == (psU32)-1) { + psError(PS_ERR_UNKNOWN, false, "error resolving error code"); + return false; + } +#endif + + // update the database + psMetadata *values = psMetadataAlloc(); + if (!psMetadataAddS16(values, PS_LIST_HEAD, "fault", 0, NULL, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to add metadata item fault"); + psFree(values); + return false; + } + + long rowsAffected = psDBUpdateRows(dbh, tableName, where, values); + psFree(values); + if (rowsAffected < 0) { + // database error + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (rowsAffected < 1) { + // we didn't do anything + psError(PS_ERR_UNKNOWN, false, "zero rows were affected - either the search criteria didn't match any rows or the field already had the value being set."); + return false; + } + + return true; +} Index: /branches/jh_branches/backtrack/ippTools/src/pxinject.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxinject.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxinject.c (revision 22250) @@ -0,0 +1,268 @@ +/* + * pxinject.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxinject.h" + +static bool newExpMode(pxConfig *config); +static bool newImfileMode(pxConfig *config); +static bool updatenewExpMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pxinjectConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(PXINJECT_MODE_NEWEXP, newExpMode); + MODECASE(PXINJECT_MODE_NEWIMFILE, newImfileMode); + MODECASE(PXINJECT_MODE_UPDATENEWEXP, updatenewExpMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint (stderr, "failure\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool newExpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id"); + return false; + } + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "-exp_id is required"); + return false; + } + psString camera = psMetadataLookupStr(&status, config->args, "-inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst"); + return false; + } + if (!camera) { + psError(PS_ERR_UNKNOWN, true, "-inst is required"); + return false; + } + psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope"); + return false; + } + if (!telescope) { + psError(PS_ERR_UNKNOWN, true, "-telescope is required"); + return false; + } + psS32 imfiles = psMetadataLookupS32(&status, config->args, "-imfiles"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -imfiles"); + return false; + } + if (!imfiles) { + psError(PS_ERR_UNKNOWN, true, "-imfiles is required"); + return false; + } + + psString workdir = psMetadataLookupStr(&status, config->args, "-workdir"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -workdir"); + return false; + } + if (!workdir) { + psError(PS_ERR_UNKNOWN, true, "-workdir is required"); + return false; + } + + psString exp_tag = pxGenExpTag(config, exp_id); + if (!exp_tag) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!newExpInsert(config->dbh, + exp_tag, + exp_id, + "reg", // state + camera, + telescope, + imfiles, + workdir, + "dirty" + ) + ) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(exp_tag); + return false; + } + + psMetadata *md = psMetadataAlloc(); + if (!psMetadataAddStr(md, PS_LIST_TAIL, "exp_tag", 0, NULL, exp_tag)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_tag"); + psFree(md); + psFree(exp_tag); + psFree(md); + } + psFree(exp_tag); + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(md); + return false; + } + } + + // negate simple so the default is true + if (!ippdbPrintMetadata(stdout, md, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(md); + return false; + } + + psFree(md); + + return true; +} + +static bool newImfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString exp_tag = psMetadataLookupStr(&status, config->args, "-exp_tag"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_tag"); + return false; + } + if (!exp_tag) { + psError(PS_ERR_UNKNOWN, true, "-exp_tag is required"); + return false; + } +#if 0 + psString class = psMetadataLookupStr(&status, config->args, "-class"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class"); + return false; + } + if (!class) { + psError(PS_ERR_UNKNOWN, true, "-class is required"); + return false; + } +#endif + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + return false; + } + if (!class_id) { + psError(PS_ERR_UNKNOWN, true, "-class_id is required"); + return false; + } + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + + // insert with error flag state set to 0 (no errors) + if (!newImfileInsert(config->dbh, exp_tag, class_id, uri)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updatenewExpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString exp_tag = psMetadataLookupStr(&status, config->args, "-exp_tag"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_tag"); + return false; + } + if (!exp_tag) { + psError(PS_ERR_UNKNOWN, true, "-exp_tag is required"); + return false; + } + + psString state = psMetadataLookupStr(&status, config->args, "-state"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -state"); + return false; + } + if (!state) { + psError(PS_ERR_UNKNOWN, true, "-state is required"); + return false; + } + + if (state) { + // set detRun.state to state + return pxnewExpSetState(config, exp_tag, state); + } + + return true; +} Index: /branches/jh_branches/backtrack/ippTools/src/pxinject.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxinject.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxinject.h (revision 22250) @@ -0,0 +1,34 @@ +/* + * pxinject.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXINJECT_H +#define PXINJECT_H 1 + +#include "pxtools.h" + +typedef enum { + PXINJECT_MODE_NONE = 0x0, + PXINJECT_MODE_NEWEXP, + PXINJECT_MODE_NEWIMFILE, + PXINJECT_MODE_UPDATENEWEXP +} pxinjectMode; + +pxConfig *pxinjectConfig(pxConfig *config, int argc, char **argv); + +#endif // PXINJECT_H Index: /branches/jh_branches/backtrack/ippTools/src/pxinjectConfig.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxinjectConfig.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxinjectConfig.c (revision 22250) @@ -0,0 +1,198 @@ +/* + * pxinjectConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pxinject.h" + +pxConfig *pxinjectConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -newExp + psMetadata *newExpArgs = psMetadataAlloc(); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-exp_id", 0, + "define the exp_id (required)", NULL); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-inst", 0, + "define the camera name (required)", NULL); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-telescope", 0, + "define the telescope name (required)", NULL); + psMetadataAddS32(newExpArgs, PS_LIST_TAIL, "-imfiles", 0, + "define the number of imfiles in this exp (required)", 0); + psMetadataAddStr(newExpArgs, PS_LIST_TAIL, "-workdir", 0, + "define workdir (required)", 0); + psMetadataAddBool(newExpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -newImfile + psMetadata *newImfileArgs = psMetadataAlloc(); + psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-exp_tag", 0, + "define the exp_tag (required)", NULL); +#if 0 + psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-class", 0, + "define the class (required)", NULL); +#endif + psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-class_id", 0, + "define the class ID (required)", NULL); + psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-uri", 0, + "define the URI (required)", NULL); + + // -updatenewExp + psMetadata *updatenewExpArgs = psMetadataAlloc(); + psMetadataAddStr(updatenewExpArgs, PS_LIST_TAIL, "-exp_tag", 0, + "define the exp_tag (required)", NULL); + psMetadataAddStr(updatenewExpArgs, PS_LIST_TAIL, "-state", 0, + "set state (required)", NULL); + + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXTOOL_ADD_MODE("-newExp", "", PXINJECT_MODE_NEWEXP, newExpArgs); + PXTOOL_ADD_MODE("-newImfile", "", PXINJECT_MODE_NEWIMFILE, newImfileArgs); + PXTOOL_ADD_MODE("-updatenewExp", "", PXINJECT_MODE_UPDATENEWEXP, updatenewExpArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // setup search criterion +#define addWhereStr(name) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, #name, 0, "==", str)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + + // generate SQL where clause + config->where = psMetadataAlloc(); + + addWhereStr(det_id); + { + int n = 0; + bool status = false; + if ((n = psMetadataLookupS32(&status, config->args, "-iteration"))) { + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, "iteration", 0, "==", n)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item iteration"); + psFree(config); + return NULL; + } + } + } + addWhereStr(det_type); + addWhereStr(exp_tag); + addWhereStr(class_id); + // convert '-inst' to 'camera' + { + psString str = NULL; + bool status = false; + if ((str = psMetadataLookupStr(&status, config->args, "-inst"))) { + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, "camera", 0, "==", str)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); + psFree(config); + return NULL; + } + } + } + addWhereStr(telescope); + { + int n = 0; + bool status = false; + if ((n = psMetadataLookupS32(&status, config->args, "-imfiles"))) { + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, "imfiles", 0, "==", n)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item imfiles"); + psFree(config); + return NULL; + } + } + } + addWhereStr(filter); + addWhereStr(stats); + addWhereStr(recipe); + { + int n = 0; + bool status = false; + if ((n = psMetadataLookupS32(&status, config->args, "-guide_version"))) { + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, "guide_version", 0, "==", n)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item guide_version"); + psFree(config); + return NULL; + } + } + } + { + bool boolean = false; + bool status = false; + + if ((boolean = psMetadataLookupBool(&status, config->args, "-accept"))) { + if (!psMetadataAddBool(config->where, PS_LIST_TAIL, "accept", 0, "==", boolean)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item accept"); + psFree(config); + return NULL; + } + } + } + + if (config->where->list->n < 1) { + psFree(config->where); + config->where = NULL; + } + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = pmConfigDB(config->modules); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/jh_branches/backtrack/ippTools/src/pxio.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxio.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxio.c (revision 22250) @@ -0,0 +1,59 @@ +/* + * pxio.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "pxio.h" + +bool convertIdToStr(psArray *mds) +{ + PS_ASSERT_PTR_NON_NULL(mds, false); + + char *ids[] = {"guide_id", "chip_id", "cam_id", "det_id", "warp_id", "diff_id", "stack_id", NULL}; + + // loop over array of metadatas + for (long i = 0; i < psArrayLength(mds); i++) { + psMetadata *md = mds->data[i]; + // loop over array of ID fields + for (int j = 0; ids[j] != NULL; j++) { + char *name = ids[j]; + bool status = false; + psS64 numeric_id = psMetadataLookupS64(&status, md, name); + if (!status) { + continue; +// psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", +// name); + // return false; + } + psMetadataRemoveKey(md, name); + psString string_id = psDBIntToString((psU64)numeric_id); + psMetadataAddStr(md, PS_LIST_HEAD, name, 0, NULL, string_id); + psFree(string_id); + } + } + + return true; +} Index: /branches/jh_branches/backtrack/ippTools/src/pxio.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxio.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxio.h (revision 22250) @@ -0,0 +1,27 @@ +/* + * pxio.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXIO_H +#define PXIO_H 1 + +#include + +bool convertIdToStr(psArray *mds); + +#endif // PXIO_H Index: /branches/jh_branches/backtrack/ippTools/src/pxregister.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxregister.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxregister.c (revision 22250) @@ -0,0 +1,56 @@ +/* + * register.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxregister.h" + +bool pxnewExpSetState(pxConfig *config, const char *exp_tag, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(exp_tag, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "reg", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid newExp state: %s", state); + return false; + } + + char *query = "UPDATE newExp SET state = '%s' WHERE exp_tag = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, state, exp_tag)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for exp_tag %s", exp_tag); + return false; + } + + return true; +} Index: /branches/jh_branches/backtrack/ippTools/src/pxregister.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxregister.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxregister.h (revision 22250) @@ -0,0 +1,29 @@ +/* + * pxregister.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXREGISTER_H +#define PXREGISTER_H 1 + +#include + +#include "pxtools.h" + +bool pxnewExpSetState(pxConfig *config, const char *exp_tag, const char *state); + +#endif // PXREGISTER_H Index: /branches/jh_branches/backtrack/ippTools/src/pxtables.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxtables.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxtables.c (revision 22250) @@ -0,0 +1,179 @@ +/* + * pxtables.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include "pxtools.h" + +#define CREATE_TABLE(createFunc) \ + if (!createFunc(config->dbh)) { \ + psError(PS_ERR_UNKNOWN, false, "dbh access failed"); \ + status = false; \ + } + +#define DROP_TABLE(dropFunc) \ + if (!dropFunc(config->dbh)) { \ + psError(PS_ERR_UNKNOWN, false, "dbh access failed"); \ + status = false; \ + } + +bool pxCreateTables(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = true; + CREATE_TABLE(expTagCounterCreateTable); + // the counter must be initialized to 0 + if (!p_psDBRunQuery(config->dbh, "INSERT INTO expTagCounter VALUES (0)")) { + psError(PS_ERR_UNKNOWN, false, "database error"); + status = false; + } + CREATE_TABLE(summitExpCreateTable); + CREATE_TABLE(summitImfileCreateTable); + CREATE_TABLE(pzPendingExpCreateTable); + CREATE_TABLE(pzPendingImfileCreateTable); + CREATE_TABLE(pzDoneExpCreateTable); + CREATE_TABLE(pzDoneImfileCreateTable); + CREATE_TABLE(newExpCreateTable); + CREATE_TABLE(newImfileCreateTable); + CREATE_TABLE(rawExpCreateTable); + CREATE_TABLE(rawImfileCreateTable); + CREATE_TABLE(guidePendingExpCreateTable); + CREATE_TABLE(chipRunCreateTable); + CREATE_TABLE(chipInputImfileCreateTable); + CREATE_TABLE(chipProcessedImfileCreateTable); + CREATE_TABLE(chipMaskCreateTable); + CREATE_TABLE(camRunCreateTable); + CREATE_TABLE(camProcessedExpCreateTable); + CREATE_TABLE(camMaskCreateTable); + CREATE_TABLE(detRunCreateTable); + CREATE_TABLE(detInputExpCreateTable); + CREATE_TABLE(detProcessedImfileCreateTable); + CREATE_TABLE(detProcessedExpCreateTable); + CREATE_TABLE(detStackedImfileCreateTable); + CREATE_TABLE(detNormalizedStatImfileCreateTable); + CREATE_TABLE(detNormalizedImfileCreateTable); + CREATE_TABLE(detNormalizedExpCreateTable); + CREATE_TABLE(detResidImfileCreateTable); + CREATE_TABLE(detResidExpCreateTable); + CREATE_TABLE(detRunSummaryCreateTable); + CREATE_TABLE(warpRunCreateTable); + CREATE_TABLE(warpInputExpCreateTable); + CREATE_TABLE(warpSkyCellMapCreateTable); + CREATE_TABLE(warpSkyfileCreateTable); + CREATE_TABLE(diffRunCreateTable); + CREATE_TABLE(diffInputSkyfileCreateTable); + CREATE_TABLE(diffSkyfileCreateTable); + CREATE_TABLE(stackRunCreateTable); + CREATE_TABLE(stackInputSkyfileCreateTable); + CREATE_TABLE(stackSumSkyfileCreateTable); + + return true; +} + +bool pxDeleteTables(pxConfig *config) { + bool status = true; + char line[128], answer[128]; + + PS_ASSERT_PTR_NON_NULL(config, false); + + fprintf (stdout, "*** delete the chip tables? ***\n"); + fprintf (stdout, "*** to delete the tables, answer YES, and give password ***\n"); + fprintf (stdout, "*** WARNING: this action is permanent ***\n\n"); + + fprintf (stdout, "delete the chip tables (YES/[n]): "); + fgets (line, 128, stdin); + sscanf (line, "%s", answer); + if (strcmp (answer, "YES")) { + psError(PS_ERR_UNKNOWN, true, "tables NOT deleleted"); + return false; + } + + fprintf (stdout, "enter dbh connection password: "); + fgets (line, 128, stdin); + sscanf (line, "%s", answer); + + { + bool status; + psString dbPassword = psMetadataLookupStr(&status, config->modules->site, "DBPASSWORD"); + if (strcmp (answer, dbPassword)) { + psError(PS_ERR_UNKNOWN, true, "tables NOT deleleted"); + return false; + } + } + + // disable foreign key constrants + if (!p_psDBRunQuery(config->dbh, "SET FOREIGN_KEY_CHECKS=0")) { + psError(PS_ERR_UNKNOWN, false, "database error"); + status = false; + } + + DROP_TABLE(expTagCounterDropTable); + DROP_TABLE(summitExpDropTable); + DROP_TABLE(summitImfileDropTable); + DROP_TABLE(pzPendingExpDropTable); + DROP_TABLE(pzPendingImfileDropTable); + DROP_TABLE(pzDoneExpDropTable); + DROP_TABLE(pzDoneImfileDropTable); + DROP_TABLE(newExpDropTable); + DROP_TABLE(newImfileDropTable); + DROP_TABLE(rawExpDropTable); + DROP_TABLE(rawImfileDropTable); + DROP_TABLE(guidePendingExpDropTable); + DROP_TABLE(chipRunDropTable); + DROP_TABLE(chipInputImfileDropTable); + DROP_TABLE(chipProcessedImfileDropTable); + DROP_TABLE(chipMaskDropTable); + DROP_TABLE(camRunDropTable); + DROP_TABLE(camProcessedExpDropTable); + DROP_TABLE(camMaskDropTable); + DROP_TABLE(detRunDropTable); + DROP_TABLE(detInputExpDropTable); + DROP_TABLE(detProcessedImfileDropTable); + DROP_TABLE(detProcessedExpDropTable); + DROP_TABLE(detStackedImfileDropTable); + DROP_TABLE(detNormalizedStatImfileDropTable); + DROP_TABLE(detNormalizedImfileDropTable); + DROP_TABLE(detNormalizedExpDropTable); + DROP_TABLE(detResidImfileDropTable); + DROP_TABLE(detResidExpDropTable); + DROP_TABLE(detRunSummaryDropTable); + DROP_TABLE(warpRunDropTable); + DROP_TABLE(warpInputExpDropTable); + DROP_TABLE(warpSkyCellMapDropTable); + DROP_TABLE(warpSkyfileDropTable); + DROP_TABLE(diffRunDropTable); + DROP_TABLE(diffInputSkyfileDropTable); + DROP_TABLE(diffSkyfileDropTable); + DROP_TABLE(stackRunDropTable); + DROP_TABLE(stackInputSkyfileDropTable); + DROP_TABLE(stackSumSkyfileDropTable); + + // re-enable foreign key constrants + if (!p_psDBRunQuery(config->dbh, "SET FOREIGN_KEY_CHECKS=1")) { + psError(PS_ERR_UNKNOWN, false, "database error"); + status = false; + } + + return status; +} Index: /branches/jh_branches/backtrack/ippTools/src/pxtag.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxtag.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxtag.c (revision 22250) @@ -0,0 +1,102 @@ +/* + * pxtag.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pxtag.h" + +psString pxGenExpTag(pxConfig *config, const char *exp_id) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // start a transaction so we don't increment the expTag counter unless we + // can successfully retreive it's value + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return NULL; + } + + if (!p_psDBRunQuery(config->dbh, + "UPDATE expTagCounter SET counter = LAST_INSERT_ID(counter + 1)")) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return NULL; + } + + // psDBLastInsertID() can't be used here as it called mysql_insert_id() + // which doesn't work with this trick. See: + // http://dev.mysql.com/doc/refman/4.1/en/information-functions.html + if (!p_psDBRunQuery(config->dbh, "SELECT LAST_INSERT_ID() as counter")) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return NULL; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return NULL; + } + // sanity check that we only got one row + if (psArrayLength(output) != 1) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "should have gotten 1 row but %lu rows were returned", psArrayLength(output)); + psFree(output); + return NULL; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return NULL; + } + + psMetadata *row = output->data[0]; + bool status = false; + psU64 counter = psMetadataLookupU64(&status, row, "counter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for counter"); + psFree(output); + return NULL; + } + psString exp_tag = NULL; + psStringAppend(&exp_tag, "%s.%" PRIu64, exp_id, counter); + psFree(output); + + return exp_tag; +} Index: /branches/jh_branches/backtrack/ippTools/src/pxtag.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxtag.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxtag.h (revision 22250) @@ -0,0 +1,29 @@ +/* + * pxtag.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXTAG_H +#define PXTAG_H 1 + +#include + +#include "pxconfig.h" + +psString pxGenExpTag(pxConfig *config, const char *exp_id); + +#endif // PXTAG_H Index: /branches/jh_branches/backtrack/ippTools/src/pxtools.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxtools.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxtools.h (revision 22250) @@ -0,0 +1,64 @@ +/* + * pxtools.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXTOOLS_H +#define PXTOOLS_H 1 + +#include +#include // for strcasecmp +#include // for unlink +#include +#include +#include + +#include "pxtoolsErrorCodes.h" +#include "pxio.h" +#include "pxdata.h" +#include "pxtag.h" +#include "pxregister.h" +#include "pxchip.h" +#include "pxcam.h" +#include "pxconfig.h" + +# define MAX_ROWS 10e9 +# define PXTOOL_MODE_NONE 0x0 + +bool pxCreateTables (pxConfig *config); +bool pxDeleteTables (pxConfig *config); + +bool pxSetFaultCode(psDB *dbh, const char *tableName, psMetadata *where, psS16 code); + +psExit pxerrorGetExitStatus (); + +void pxUsage(FILE *stream, int argc, char **argv, const char *modeName, psMetadata *argSet); +bool pxGetOptions(FILE *stream, int argc, char **argv, pxConfig *config, psMetadata *modes, psMetadata *argSets); + +#define PXTOOL_ADD_MODE(option, comment, modeval, argset) \ +{ \ + if (!psMetadataAddMetadata(argSets, PS_LIST_TAIL, option, 0, comment, argset)) {;\ + psError(PS_ERR_UNKNOWN, false, "failed to add argset for %s", option); \ + } \ + psFree(argset); \ +\ + if (!psMetadataAddU32(modes, PS_LIST_TAIL, option, 0, comment, modeval)) {;\ + psError(PS_ERR_UNKNOWN, false, "failed to add argset for %s", option); \ + } \ +} + +#endif // PXTOOLS_H Index: /branches/jh_branches/backtrack/ippTools/src/pxtoolsErrorCodes.c.in =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxtoolsErrorCodes.c.in (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxtoolsErrorCodes.c.in (revision 22250) @@ -0,0 +1,25 @@ +/* + * The line + { PXTOOLS_ERR_$X{ErrorCode}, "$X{ErrorDescription}"}, + * (without the Xs) + * will be replaced by values from errorCodes.dat + */ +#include "pslib.h" +#include "pxtoolsErrorCodes.h" + +void pxtoolsErrorRegister(void) +{ + static psErrorDescription errors[] = { + { PXTOOLS_ERR_BASE, "First value we use; lower values belong to psLib" }, + { PXTOOLS_ERR_${ErrorCode}, "${ErrorDescription}"}, + }; + static int nerror = PXTOOLS_ERR_NERROR - PXTOOLS_ERR_BASE; // number of values in enum + + for (int i = 0; i < nerror; i++) { + psErrorDescription *tmp = psAlloc(sizeof(psErrorDescription)); + *tmp = errors[i]; + psErrorRegister(tmp, 1); + psFree(tmp); /* it's on the internal list */ + } + nerror = 0; // don't register more than once +} Index: /branches/jh_branches/backtrack/ippTools/src/pxtoolsErrorCodes.dat =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxtoolsErrorCodes.dat (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxtoolsErrorCodes.dat (revision 22250) @@ -0,0 +1,10 @@ +# +# This file is used to generate pxtoolsErrorCodes.h +# +BASE = 1100 First value we use; lower values belong to psLib +UNKNOWN Unknown PM error code +ARGUMENTS Incorrect arguments +SYS System error +CONFIG Problem in configure files +PROG Programming error +DATA invalid data Index: /branches/jh_branches/backtrack/ippTools/src/pxtoolsErrorCodes.h.in =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pxtoolsErrorCodes.h.in (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pxtoolsErrorCodes.h.in (revision 22250) @@ -0,0 +1,18 @@ +#if !defined(PXTOOLS_ERROR_CODES_H) +#define PXTOOLS_ERROR_CODES_H +/* + * The line + * PXTOOLS_ERR_$X{ErrorCode}, + * (without the X) + * + * will be replaced by values from errorCodes.dat + */ +typedef enum { + PXTOOLS_ERR_BASE = 512, + PXTOOLS_ERR_${ErrorCode}, + PXTOOLS_ERR_NERROR +} pxtoolsErrorCode; + +void pxtoolsErrorRegister(void); + +#endif Index: /branches/jh_branches/backtrack/ippTools/src/pzgetexp.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pzgetexp.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pzgetexp.c (revision 22250) @@ -0,0 +1,372 @@ +/* + * pzgetexp.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pzgetexp.h" + +#define PRODUCT_LS_CMD "dsproductls" + +static bool go (pxConfig *config); +static psArray *parseFileSets(pxConfig *config, const char *str); + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pzgetexpConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + if (!go(config)) { + goto FAIL; + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool go(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + // find last fileset/exp_id (if we have one) + bool haveLastFileSet = false; + psString lastFileSet = NULL; + { + char *query = "SELECT * from summitExp ORDER BY dateobs DESC LIMIT 1"; + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, false, "no summitExp rows found"); + haveLastFileSet = false; + psFree(output); + } else { + haveLastFileSet = true; + bool status = false; + lastFileSet = psStringCopy(psMetadataLookupStr(&status, output->data[0], "exp_id")); + psFree(output); + } + } + + // invoke dsproductls + // dsproductls --uri <> --last_fileset <> + bool status = false; + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + psString cmd = NULL; + if (haveLastFileSet) { + psStringAppend(&cmd, "%s --uri %s --last_fileset %s", + PRODUCT_LS_CMD, uri, lastFileSet); + psFree(lastFileSet); + } else { + psStringAppend(&cmd, "%s --uri %s", PRODUCT_LS_CMD, uri); + } + + psTrace("pzgetexp", PS_LOG_INFO, "cmd is: %s\n", cmd); + + FILE *output = popen(cmd, "r"); + psFree(cmd); + + if (!output) { + psError(PS_ERR_UNKNOWN, true, "popen() failed"); + return false; + } + psString cmdOutput = psSlurpFile(output); + pclose(output); + + psArray *newSummitExps = parseFileSets(config, cmdOutput); + psFree(cmdOutput); + if (!newSummitExps) { + // XXX not nessicarily an error + psError(PS_ERR_UNKNOWN, true, "no new fileSet/exp IDs"); + return false; + } + if (!psArrayLength(newSummitExps)) { + psTrace("pzgetexp", PS_LOG_INFO, "no new fileSet/exp IDs"); + psFree(newSummitExps); + return true; + } + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // create a temporry table + { + char *query = "CREATE TEMPORARY TABLE incoming" + " (exp_id VARCHAR(64), camera VARCHAR(64), telescope VARCHAR(64), dateobs DATETIME, exp_type VARCHAR(64), uri VARCHAR(255), PRIMARY KEY(exp_id, camera, telescope))" + " ENGINE=MEMORY"; + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newSummitExps); + return false; + } + } + + { + char *query = "INSERT INTO incoming (exp_id, camera, telescope, dateobs, exp_type, uri) VALUES (?, ?, ?, ?, ?, ?)"; + + long inserted = p_psDBRunQueryPrepared(config->dbh, newSummitExps, query); + if (inserted < 0) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newSummitExps); + return false; + } + // sanity check that we actually inserted something + if (inserted == 0) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error -- we should have inserted at least one row"); + psFree(newSummitExps); + return false; + } + } + + psFree(newSummitExps); + + // add new exps to pzPendingExp -- must be done before the new exps are + // added to summitExp because of the SQL logic + { + char *query = + "INSERT INTO pzPendingExp" + " SElECT" + " incoming.exp_id," + " incoming.camera," + " incoming.telescope" + " FROM incoming" + " LEFT JOIN summitExp" + " USING(exp_id, camera, telescope)" + " WHERE" + " summitExp.exp_id is NULL" + " AND summitExp.camera is NULL" + " AND summitExp.telescope is NULL"; + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // add new exps to summitExp + { + char *query = + "INSERT INTO summitExp" + " SElECT" + " incoming.*," + " NULL" + " FROM incoming" + " LEFT JOIN summitExp" + " USING(exp_id, camera, telescope)" + " WHERE" + " summitExp.exp_id is NULL" + " AND summitExp.camera is NULL" + " AND summitExp.telescope is NULL"; + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static psArray *parseFileSets(pxConfig *config, const char *str) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(str, NULL); + + // these are constants for all records parsed -- look them up before we do + // any work + bool status = false; + char *camera = psMetadataLookupStr(&status, config->args, "-inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup item '-inst'"); + return NULL; + } + char *telescope = psMetadataLookupStr(&status, config->args, "-telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup item '-telescope'"); + return NULL; + } + + // split the string into lines + psList *doc = psStringSplit(str, "\n", false); + + psListIterator *lineCursor = psListIteratorAlloc(doc, 0, false); + + psArray *summitExps = psArrayAllocEmpty(psListLength(doc)); + psString line; + while ((line = psListGetAndIncrement(lineCursor))) { + psTrace("pzgetimfile", PS_LOG_INFO, "parsing line: %s\n", line); + + // split line into tokens + psList *tokens = psStringSplit(line, " ", false); + + // check to see if this line is a comment (or if the first token is + // NULL) + if (!psListGet(tokens, 0) || *((char *)psListGet(tokens, 0)) == '#') { + psFree(tokens); + continue; + } + + // check that we have the right number of tokens + // print "# uri fileset datetime type\n"; + if (psListLength(tokens) != 4) { + // error + return false; + } + + // find the values of interest + psListIterator *tokenCursor = psListIteratorAlloc(tokens, 0, false); + char *uri = psListGetAndIncrement(tokenCursor); + char *exp_id = psListGetAndIncrement(tokenCursor); // fileset + char *dateobsStr= psListGetAndIncrement(tokenCursor); // datetime + char *exp_type = psListGetAndIncrement(tokenCursor); // type + + // create a new metadata to represent this line and it's values + psMetadata *md = psMetadataAlloc(); + if (!psMetadataAddStr(md, PS_LIST_TAIL, "exp_id", 0, NULL, exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "camera", 0, NULL, camera)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "telescope", 0, NULL, telescope)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item telescope"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + psTime *dateobs = psTimeFromISO(dateobsStr, PS_TIME_UTC); + if (!psMetadataAddTime(md, PS_LIST_TAIL, "dateobs", 0, NULL, dateobs)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item telescope"); + psFree(dateobs); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + psFree(dateobs); + if (!psMetadataAddStr(md, PS_LIST_TAIL, "exp_type", 0, NULL, exp_type)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_type"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "uri", 0, NULL, uri)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item uri"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + + // must be freed after the new metadata is built -- holds the strings + psFree(tokenCursor); + psFree(tokens); + + // add the new metadata to the result set + psArrayAdd(summitExps, 0, md); + + // debugging + if (psTraceGetLevel("pzgetexp") == PS_LOG_INFO) { + psString doc = psMetadataConfigFormat(md); + psTrace("pzgetexp", PS_LOG_INFO, "parsed line as:\n %s\n", doc); + psFree(doc); + } + + psFree(md); + + } + + psFree(lineCursor); + psFree(doc); + + return summitExps; +} Index: /branches/jh_branches/backtrack/ippTools/src/pzgetexp.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pzgetexp.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pzgetexp.h (revision 22250) @@ -0,0 +1,27 @@ +/* + * pzgetexp.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PZGETEXP_H +#define PZGETEXP_H 1 + +#include "pxtools.h" + +pxConfig *pzgetexpConfig(pxConfig *config, int argc, char **argv); + +#endif // PZGETEXP_H Index: /branches/jh_branches/backtrack/ippTools/src/pzgetexpConfig.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pzgetexpConfig.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pzgetexpConfig.c (revision 22250) @@ -0,0 +1,83 @@ +/* + * pzgetexpConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pxtools.h" + +pxConfig *pzgetexpConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); + psFree(config); + return NULL; + } + + psMetadata *args = psMetadataAlloc(); + psMetadataAddStr(args , PS_LIST_TAIL, "-uri", 0, + "DataStore product URI", ""); + psMetadataAddStr(args , PS_LIST_TAIL, "-inst", 0, + "camera name", ""); + psMetadataAddStr(args , PS_LIST_TAIL, "-telescope", 0, + "telescope name", ""); + + bool status = false; + if (!psArgumentParse(args, &argc, argv) + || argc != 1 + || strcmp(psMetadataLookupStr(&status, args, "-uri"), "") == 0 + || strcmp(psMetadataLookupStr(&status, args, "-inst"), "") == 0 + || strcmp(psMetadataLookupStr(&status, args, "-telescope"), "") == 0 + ) { + fprintf(stderr, "error parsing arguments\n"); + printf("\nPan-STARRS Phase Z Search Tool\n"); + printf("Usage: %s -uri -inst -telescope \n\n", + argv[0]); + psArgumentHelp(args); + psFree(config); + return NULL; + } + + config->args = args; + // don't free args here as it's silly to increment the ref count then + // "free" it + + // define Database handle, if used + config->dbh = pmConfigDB(config->modules); + if(!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't connect to db\n"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/jh_branches/backtrack/ippTools/src/pzgetimfiles.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pzgetimfiles.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pzgetimfiles.c (revision 22250) @@ -0,0 +1,682 @@ +/* + * pzgetimfiles.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtag.h" +#include "pxtools.h" +#include "pzgetimfiles.h" + +#define FILESET_LS_CMD "dsfilesetls" + +static bool go (pxConfig *config); +static psArray *parseFiles(pxConfig *config, const char *str); + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pzgetimfilesConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + if (!go(config)) { + goto FAIL; + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool go(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + bool status = false; + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + + psString filesetid = psMetadataLookupStr(&status, config->args, "-filesetid"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filesetid"); + return false; + } + if (!filesetid) { + psError(PS_ERR_UNKNOWN, true, "-filesetid is required"); + return false; + } + + psString camera = psMetadataLookupStr(&status, config->args, "-inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst"); + return NULL; + } + if (!camera) { + psError(PS_ERR_UNKNOWN, true, "-inst is required"); + return NULL; + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope"); + return NULL; + } + if (!telescope) { + psError(PS_ERR_UNKNOWN, true, "-telescope is required"); + return NULL; + } + + // invoke dsfilesetls + psString cmd = NULL; + psStringAppend(&cmd, "%s --uri %s", FILESET_LS_CMD, uri); + + psTrace("pzgetimfiles", PS_LOG_INFO, "cmd is: %s\n", cmd); + + FILE *output = popen(cmd, "r"); + psFree(cmd); + + if (!output) { + psError(PS_ERR_UNKNOWN, true, "popen() failed"); + return false; + } + + psString cmdOutput = psSlurpFile(output); + int exitStatus = pclose(output); + + if (exitStatus != 0) { + psError(PS_ERR_UNKNOWN, true, "%s failed with exit status %d", + FILESET_LS_CMD, exitStatus); + psFree(cmdOutput); + return false; + } + + // prase output of dsfilesetls + psArray *newImfiles = parseFiles(config, cmdOutput); + if (!newImfiles) { + // XXX not nessicarily an error + psError(PS_ERR_UNKNOWN, true, "no new files/imfiles"); + psFree(cmdOutput); + return false; + } + psFree(cmdOutput); + + // save the number of new Imfiles; + long imfiles = psArrayLength(newImfiles); + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // if the fileset was empty (no files) then we can bail out early + if (imfiles == 0) { + psFree(newImfiles); + + char *query = + "UPDATE summitExp" + " SET imfiles = %d" + " WHERE exp_id = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, imfiles, filesetid, camera, telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // remove the pzPendingExp entry for this exp (fileset) + { + char *query = + "DELETE FROM pzPendingExp" + " WHERE" + " exp_id = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, filesetid, camera, telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // sanity check: we should have removed only one row + psU64 affected = psDBAffectedRows(config->dbh); + if (psDBAffectedRows(config->dbh) != 1) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row but %" PRIu64 " rows were modified", affected); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; + } + + // try not to insert duplicate pzPendingExp/pzPendingImfile entries + + // create a temp table + { + char *query = + "CREATE TEMPORARY TABLE incoming (exp_id VARCHAR(64), camera VARCHAR(64), telescope VARCHAR(64), file_id VARCHAR(64), bytes INT, md5sum VARCHAR(32), class VARCHAR(64), class_id VARCHAR(64), uri VARCHAR(255), PRIMARY KEY(exp_id, camera, telescope, class, class_id)) ENGINE=MEMORY"; + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newImfiles); + return false; + } + } + + // load the imfiles (files) into the temp table + { + char *query = "INSERT INTO incoming (exp_id, camera, telescope, file_id, bytes, md5sum, class, class_id, uri) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"; + + long inserted = p_psDBRunQueryPrepared(config->dbh, newImfiles, query); + if (inserted < 0) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(newImfiles); + return false; + } + // sanity check that we actually inserted something + if (inserted == 0) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error -- we should have inserted at least one row"); + psFree(newImfiles); + return false; + } + } + + psFree(newImfiles); + + // queue the imfiles (files) in pzPendingImfile -- must be done before the + // imfile is insterted into summitImfile (after that happens when don't + // know which imfiles are new anymore (well thats not exactly true but it + // make be someday if we have to beable to resync to modified datastore + // entries) + { + char *query = + "INSERT INTO pzPendingImfile" + " SElECT" + " incoming.exp_id," + " incoming.camera," + " incoming.telescope," + " incoming.class," + " incoming.class_id," + " NULL" // exp_tag, will be assigned later + " FROM incoming" + " JOIN pzPendingExp" + " USING(exp_id, camera, telescope)" + " LEFT JOIN summitImfile" + " USING(exp_id, camera, telescope, class, class_id)" + " WHERE" + " summitImfile.exp_id is NULL" + " AND summitImfile.camera is NULL" + " AND summitImfile.telescope is NULL" + " AND summitImfile.class is NULL" + " AND summitImfile.class_id is NULL"; + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // copy imfiles (files) from the temp table into summitImfiles + { + char *query = + "INSERT INTO summitImfile" + " SElECT" + " incoming.exp_id," + " incoming.camera," + " incoming.telescope," + " incoming.file_id," + " incoming.bytes," + " incoming.md5sum," + " incoming.class," + " incoming.class_id," + " incoming.uri" + " FROM incoming" + " LEFT JOIN summitImfile" + " USING(exp_id, camera, telescope, class, class_id)" + " WHERE" + " summitImfile.exp_id is NULL" + " AND summitImfile.camera is NULL" + " AND summitImfile.telescope is NULL" + " AND summitImfile.class is NULL" + " AND summitImfile.class_id is NULL"; + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // update summitExp.imfiles (should have been NULL) + { + char *query = + "UPDATE summitExp" + " SET imfiles = (SELECT COUNT(*) FROM summitImfile" + " WHERE" + " exp_id = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'" + ")" + " WHERE" + " exp_id = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, filesetid, camera, telescope, filesetid, camera, telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // sanity check: we should have updated only one row + psU64 affected = psDBAffectedRows(config->dbh); + if (psDBAffectedRows(config->dbh) != 1) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row but %" PRIu64 " rows were modified", affected); + return false; + } + } + + // copy the summitExp row into newExp in order to create a new exp_tag + { + psString exp_tag = pxGenExpTag(config, filesetid); + if (!exp_tag) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + char *query = + "INSERT INTO newExp" + " SElECT" + " '%s'," // exp_tag + " summitExp.exp_id," + " summitExp.camera," + " summitExp.telescope," + " summitExp.dateobs," + " summitExp.exp_type," + " summitExp.imfiles," + " 0" // error flags + " FROM summitExp" + " WHERE" + " summitExp.exp_id = '%s'" + " AND summitExp.camera = '%s'" + " AND summitExp.telescope = '%s'"; + + if (!p_psDBRunQuery(config->dbh, query, exp_tag, filesetid, camera, telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(exp_tag); + return false; + } + psFree(exp_tag); + + // sanity check: we should have inserted only one row + psU64 affected = psDBAffectedRows(config->dbh); + if (psDBAffectedRows(config->dbh) != 1) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row but %" PRIu64 " rows were modified", affected); + return false; + } + } + + // set the exp_tag for the imfiles (files) in the exposure (fileset) that + // we are investigating + { + char *query = + "UPDATE pzPendingImfile" + " SET exp_tag = (SELECT exp_tag from newExp" + " WHERE exp_id = '%s'" + " AND camera = '%s'" + " AND telescope = '%s')" + " WHERE" + " exp_id = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, filesetid, camera, telescope, filesetid, camera, telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + // cp the pzPendingExp entry to pzDoneExp + { + char *query = + "INSERT INTO pzDoneExp" + " SELECT" + " pzPendingExp.*" + " FROM pzPendingExp" + " WHERE" + " pzPendingExp.exp_id = '%s'" + " AND pzPendingExp.camera = '%s'" + " AND pzPendingExp.telescope = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, filesetid, camera, telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // sanity check: we should have inserted only one row + psU64 affected = psDBAffectedRows(config->dbh); + if (psDBAffectedRows(config->dbh) != 1) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row but %" PRIu64 " rows were modified", affected); + return false; + } + } + + // remove the pzPendingExp entry for this exp (fileset) + { + char *query = + "DELETE FROM pzPendingExp" + " WHERE" + " exp_id = '%s'" + " AND camera = '%s'" + " AND telescope = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, filesetid, camera, telescope)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // sanity check: we should have removed only one row + psU64 affected = psDBAffectedRows(config->dbh); + if (psDBAffectedRows(config->dbh) != 1) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row but %" PRIu64 " rows were modified", affected); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static psArray *parseFiles(pxConfig *config, const char *str) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(str, NULL); + + // these are constants for all records parsed -- look them up before we do + // any work + bool status = false; + psString exp_id = psMetadataLookupStr(&status, config->args, "-filesetid"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for '-filesetid'"); + return NULL; + } + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "-filesetid is required"); + return NULL; + } + + psString camera = psMetadataLookupStr(&status, config->args, "-inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst"); + return NULL; + } + if (!camera) { + psError(PS_ERR_UNKNOWN, true, "-inst is required"); + return NULL; + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope"); + return NULL; + } + if (!telescope) { + psError(PS_ERR_UNKNOWN, true, "-telescope is required"); + return NULL; + } + + // split the string into lines + psList *doc = psStringSplit(str, "\n", false); + + psListIterator *lineCursor = psListIteratorAlloc(doc, 0, false); + + psArray *pzPendingImfiles = psArrayAllocEmpty(psListLength(doc)); + psString line; + while ((line = psListGetAndIncrement(lineCursor))) { + psTrace("pzgetimfiles", PS_LOG_INFO, "parsing line: %s\n", line); + + // split line into tokens + psList *tokens = psStringSplit(line, " ", false); + + // check to see if this line is a comment (or if the first token is + // NULL) + if (!psListGet(tokens, 0) || *((char *)psListGet(tokens, 0)) == '#') { + psFree(tokens); + continue; + } + + // check that we have the right number of tokens + // print "# uri fileid bytes md5sum type \n"; + if (!psListLength(tokens) > 5) { + psError(PS_ERR_UNKNOWN, true, "invalid line format: %s", line); + psFree(tokens); + psFree(pzPendingImfiles); + psFree(lineCursor); + psFree(doc); + return false; + } + + // find the values of interest + psListIterator *tokenCursor = psListIteratorAlloc(tokens, 0, false); + char *uri = psListGetAndIncrement(tokenCursor); + char *file_id = psListGetAndIncrement(tokenCursor); // fileid + char *bytes = psListGetAndIncrement(tokenCursor); // bytes + char *md5sum = psListGetAndIncrement(tokenCursor); // md5sum + char *class = psListGetAndIncrement(tokenCursor); // type + char *class_id = psListGetAndIncrement(tokenCursor); // chipname + + // create a new metadata to represent this line and it's values + psMetadata *md = psMetadataAlloc(); + if (!psMetadataAddStr(md, PS_LIST_TAIL, "exp_id", 0, NULL, exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "camera", 0, NULL, camera)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "telescope", 0, NULL, telescope)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item telescope"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "file_id", 0, NULL, file_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item bytes"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddS64(md, PS_LIST_TAIL, "bytes", 0, NULL, (psS64)atoll(bytes))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item bytes"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "md5sum", 0, NULL, md5sum)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item md5sum"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "class", 0, NULL, class)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "class_id", 0, NULL, class_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class_id"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + if (!psMetadataAddStr(md, PS_LIST_TAIL, "uri", 0, NULL, uri)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item uri"); + psFree(md); + psFree(tokenCursor); + psFree(tokens); + return NULL; + } + + // must be freed after the new metadata is built -- holds the strings + psFree(tokenCursor); + psFree(tokens); + + // debugging + if (psTraceGetLevel("pzgetimfiles") >= PS_LOG_INFO) { + psString doc = psMetadataConfigFormat(md); + psTrace("pzgetimfiles", PS_LOG_INFO, "parsed line as:\n %s\n", doc); + psFree(doc); + } + + psArrayAdd(pzPendingImfiles, 0, md); + + psFree(md); + } + + psFree(lineCursor); + psFree(doc); + + return pzPendingImfiles;; +} Index: /branches/jh_branches/backtrack/ippTools/src/pzgetimfiles.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pzgetimfiles.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pzgetimfiles.h (revision 22250) @@ -0,0 +1,27 @@ +/* + * pzgetimfiles.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PZGETIMFILES_H +#define PZGETIMFILES 1 + +#include "pxtools.h" + +pxConfig *pzgetimfilesConfig(pxConfig *config, int argc, char **argv); + +#endif // PZGETIMFILES_H Index: /branches/jh_branches/backtrack/ippTools/src/pzgetimfilesConfig.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pzgetimfilesConfig.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pzgetimfilesConfig.c (revision 22250) @@ -0,0 +1,85 @@ +/* + * pzgetimfilesConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pxtools.h" + +pxConfig *pzgetimfilesConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); + psFree(config); + return NULL; + } + + psMetadata *args = psMetadataAlloc(); + psMetadataAddStr(args , PS_LIST_TAIL, "-uri", 0, + "DataStore FileSet URI", ""); + psMetadataAddStr(args , PS_LIST_TAIL, "-filesetid", 0, + "FileSet ID", ""); + psMetadataAddStr(args, PS_LIST_TAIL, "-inst", 0, + "search by camera", NULL); + psMetadataAddStr(args, PS_LIST_TAIL, "-telescope", 0, + "search by telescope", NULL); + + + bool status = false; + if (!psArgumentParse(args, &argc, argv) + || argc != 1 + || strcmp(psMetadataLookupStr(&status, args, "-uri"), "") == 0 + || strcmp(psMetadataLookupStr(&status, args, "-filesetid"), "") == 0 + ) { + fprintf(stderr, "error parsing arguments\n"); + printf("\nPan-STARRS Phase Z Search Tool\n"); + printf("Usage: %s -uri -filesetid \n\n", argv[0]); + psArgumentHelp(args); + psFree(args); + psFree(config); + return NULL; + } + + config->args = args; + // don't free args here as it's silly to increment the ref count then + // "free" it + + // define Database handle, if used + config->dbh = pmConfigDB(config->modules); + if(!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't connect to db\n"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/jh_branches/backtrack/ippTools/src/pztool.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pztool.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pztool.c (revision 22250) @@ -0,0 +1,444 @@ +/* + * pztool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pxtools.h" +#include "pztool.h" + +static bool seenMode(pxConfig *config); +static bool pendingExpMode(pxConfig *config); +static bool pendingImfileMode(pxConfig *config); +static bool copydoneMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pztoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(PZTOOL_MODE_SEEN, seenMode); + MODECASE(PZTOOL_MODE_PENDINGEXP, pendingExpMode); + MODECASE(PZTOOL_MODE_PENDINGIMFILE, pendingImfileMode); + MODECASE(PZTOOL_MODE_COPYDONE, copydoneMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool seenMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = psStringCopy("SELECT * FROM summitExp"); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "summitExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pztool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "summitExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool pendingExpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = psStringCopy( + "SELECT" + " summitExp.*" + " FROM pzPendingExp" + " JOIN summitExp" + " USING(exp_id, camera, telescope)" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "pzPendingExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pztool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "summitExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool pendingImfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = psStringCopy( + "SELECT" + " summitImfile.*," + " pzPendingImfile.exp_tag" + " FROM pzPendingImfile" + " JOIN summitImfile" + " USING(exp_id, camera, telescope, class, class_id)" + ); + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "pzPendingImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pztool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(output); + return false; + } + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pzPendingImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool copydoneMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id"); + return false; + } + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "-exp_id is required"); + return false; + } + + psString camera = psMetadataLookupStr(&status, config->args, "-inst"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -inst"); + return NULL; + } + if (!camera) { + psError(PS_ERR_UNKNOWN, true, "-inst is required"); + return NULL; + } + + psString telescope = psMetadataLookupStr(&status, config->args, "-telescope"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -telescope"); + return NULL; + } + if (!telescope) { + psError(PS_ERR_UNKNOWN, true, "-telescope is required"); + return NULL; + } + + psString class = psMetadataLookupStr(&status, config->args, "-class"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class"); + return NULL; + } + if (!class) { + psError(PS_ERR_UNKNOWN, true, "-class is required"); + return NULL; + } + + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + return NULL; + } + if (!class_id) { + psError(PS_ERR_UNKNOWN, true, "-class_id is required"); + return NULL; + } + + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + + // need to know exp_id, camera, telescope, class, class_id (to find the + // pzPendingImfile entry and the URI for the newImfile entry. + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // insert new imfile into newImfile + { + char *query = + "INSERT INTO newImfile" + " SElECT" + " pzPendingImfile.exp_tag," + " pzPendingImfile.class," + " pzPendingImfile.class_id," + " '%s'," // uri of downloaded file + " 0" // error flags + " FROM pzPendingImfile" + " WHERE" + " pzPendingImfile.exp_id = '%s'" + " AND pzPendingImfile.camera = '%s'" + " AND pzPendingImfile.telescope = '%s'" + " AND pzPendingImfile.class = '%s'" + " AND pzPendingImfile.class_id = '%s'"; + + if (!p_psDBRunQuery(config->dbh, query, uri, exp_id, camera, telescope, class, class_id)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // sanity check: we should have inserted only one row + psU64 affected = psDBAffectedRows(config->dbh); + if (psDBAffectedRows(config->dbh) != 1) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row but %" PRIu64 " rows were modified", affected); + return false; + } + } + + // cp the imfile into pzDoneImfile + { + char *query = + "INSERT INTO pzDoneImfile" + " SElECT" + " pzPendingImfile.*," + " '%s'" // uri of downloaded file + " FROM pzPendingImfile" + " WHERE" + " pzPendingImfile.exp_id = '%s'" + " AND pzPendingImfile.camera = '%s'" + " AND pzPendingImfile.telescope = '%s'" + " AND pzPendingImfile.class = '%s'" + " AND pzPendingImfile.class_id = '%s'"; + + if (!p_psDBRunQuery(config->dbh, query, uri, exp_id, camera, telescope, class, class_id)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // sanity check: we should have inserted only one row + psU64 affected = psDBAffectedRows(config->dbh); + if (psDBAffectedRows(config->dbh) != 1) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row but %" PRIu64 " rows were modified", affected); + return false; + } + } + + // removve the entry from pzPendingImfile + { + char *query = + "DELETE FROM pzPendingImfile" + " WHERE" + " pzPendingImfile.exp_id = '%s'" + " AND pzPendingImfile.camera = '%s'" + " AND pzPendingImfile.telescope = '%s'" + " AND pzPendingImfile.class = '%s'" + " AND pzPendingImfile.class_id = '%s'"; + + if (!p_psDBRunQuery(config->dbh, query, exp_id, camera, telescope, class, class_id)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // sanity check: we should have removed only one row + psU64 affected = psDBAffectedRows(config->dbh); + if (psDBAffectedRows(config->dbh) != 1) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row but %" PRIu64 " rows were modified", affected); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} Index: /branches/jh_branches/backtrack/ippTools/src/pztool.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pztool.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pztool.h (revision 22250) @@ -0,0 +1,35 @@ +/* + * pztool.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PZTOOL_H +#define PZTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + PZTOOL_MODE_NONE = 0x0, + PZTOOL_MODE_SEEN, + PZTOOL_MODE_PENDINGEXP, + PZTOOL_MODE_PENDINGIMFILE, + PZTOOL_MODE_COPYDONE +} pztoolMode; + +pxConfig *pztoolConfig(pxConfig *config, int argc, char **argv); + +#endif // PZTOOL_H Index: /branches/jh_branches/backtrack/ippTools/src/pztoolConfig.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/pztoolConfig.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/pztoolConfig.c (revision 22250) @@ -0,0 +1,213 @@ +/* + * pztoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pztool.h" + +pxConfig *pztoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); + psFree(config); + return NULL; + } + + // -seen + psMetadata *seenArgs = psMetadataAlloc(); + psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-exp_tag", 0, + "define exposure ID", NULL); + psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-inst", 0, + "define camera ID", NULL); + psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-telescope", 0, + "define telescope ID", NULL); + psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-exp_type", 0, + "define exposure type", NULL); + psMetadataAddBool(seenArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-exp_tag", 0, + "define exposure ID", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-inst", 0, + "define camera ID", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-telescope", 0, + "define telescope ID", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-exp_type", 0, + "define exposure type", NULL); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -pendingimfile + psMetadata *pendingimfileArgs = psMetadataAlloc(); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-exp_tag", 0, + "define exposure ID", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-inst", 0, + "define camera ID", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-telescope", 0, + "define telescope ID", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-exp_type", 0, + "define exposure type", NULL); + psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -copydone + psMetadata *copydoneArgs = psMetadataAlloc(); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-exp_id", 0, + "define exposure ID", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-inst", 0, + "define camera ID", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-telescope", 0, + "define telescope ID", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-class", 0, + "define class", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-class_id", 0, + "define class_id", NULL); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-uri", 0, + "define storage uri", NULL); + +#define PXTOOL_MODE(option, modeval, argset) \ +{ \ + int N = 0; \ + if ((N = psArgumentGet (argc, argv, option))) { \ + psArgumentRemove (N, &argc, argv); \ + if (config->mode) { \ + psError(PS_ERR_UNKNOWN, true, "only one mode selection is allowed"); \ + psFree(config); \ + return NULL; \ + } \ + config->mode = modeval; \ + config->args = psMemIncrRefCounter(argset); \ + } \ + if (!psMetadataAddMetadata(argSets, PS_LIST_TAIL, option, 0, NULL, argset)) {;\ + psError(PS_ERR_UNKNOWN, false, "failed to add argset for %s", option); \ + } \ + psFree(argset); \ +} + + psMetadata *argSets = psMetadataAlloc(); + // find which mode we're running under + PXTOOL_MODE("-seen", PZTOOL_MODE_SEEN, seenArgs); + PXTOOL_MODE("-pendingexp", PZTOOL_MODE_PENDINGEXP, pendingexpArgs); + PXTOOL_MODE("-pendingimfile", PZTOOL_MODE_PENDINGIMFILE,pendingimfileArgs); + PXTOOL_MODE("-copydone", PZTOOL_MODE_COPYDONE, copydoneArgs); + + bool argErr = false; + if (config->mode == PZTOOL_MODE_NONE) { + argErr = true; + fprintf (stderr, "mode argument is required\n"); + } else if (! psArgumentParse(config->args, &argc, argv) || argc != 1) { + argErr = true; + fprintf (stderr, "error parsing arguments\n"); + } + + if (argErr) { + printf("\nPan-STARRS Detrend Tool\n"); + printf("Usage: %s []\n\n", argv[0]); + printf(" :\n\n"); + + psMetadataIterator *iter = psMetadataIteratorAlloc(argSets, 0, NULL); + psMetadataItem *item = NULL; + while ((item = psMetadataGetAndIncrement(iter))) { + if (!item->type == PS_DATA_METADATA) { + psAbort("all options must be specified as a metadata"); + } + + fprintf(stdout, "%s ", item->name); + psArgumentHelp(item->data.md); + } + psFree(iter); + + psFree(argSets); + psFree(config); + return NULL; + } + + psFree(argSets); + + // setup search criterion +#define addWhereStr(name) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, #name, 0, "==", str)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + + // generate SQL where clause + config->where = psMetadataAlloc(); + + addWhereStr(exp_tag); + addWhereStr(telescope); + // convert '-inst' to 'camera' + { + psString str = NULL; + bool status = false; + if ((str = psMetadataLookupStr(&status, config->args, "-inst"))) { + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, "camera", 0, "==", str)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item camera"); + psFree(config); + return NULL; + } + } + } + addWhereStr(exp_type); + addWhereStr(class); + addWhereStr(class_id); + + if (config->where->list->n < 1) { + psFree(config->where); + config->where = NULL; + } + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = pmConfigDB(config->modules); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + // save argv/argc + config->argv = argv; + config->argc = argc; + + return config; +} Index: /branches/jh_branches/backtrack/ippTools/src/regtool.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/regtool.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/regtool.c (revision 22250) @@ -0,0 +1,1213 @@ +/* + * regtool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxdata.h" +#include "regtool.h" +#include "chiptool.h" + +static bool pendingimfileMode(pxConfig *config); +static bool addprocessedimfileMode(pxConfig *config); +static bool processedimfileMode(pxConfig *config); +static bool updateprocessedimfileMode(pxConfig *config); + +static bool pendingexpMode(pxConfig *config); +static bool addprocessedexpMode(pxConfig *config); +static bool processedexpMode(pxConfig *config); +static bool updateprocessedexpMode(pxConfig *config); + + +// static guidePendingExpRow *newToP1PendingExp(newExpRow *newExp); +//static chipPendingExpRow *newTochipPendingExp(pxConfig *config, newExpRow *newExp); +//static chipInputImfileRow *rawImfileTochipInputImfile(pxConfig *config, psS64 chip_id, rawImfileRow *rawImfile); + +static rawExpRow *newToRawExp(pxConfig *config, newExpRow *exp); +static rawImfileRow *newToRawImfile(pxConfig *config, newImfileRow *exp); +//static psU32 mapCodeStrToInt(const char *codeStr); + + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = regtoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(REGTOOL_MODE_PENDINGIMFILE, pendingimfileMode); + MODECASE(REGTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileMode); + MODECASE(REGTOOL_MODE_PROCESSEDIMFILE, processedimfileMode); + MODECASE(REGTOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileMode); + MODECASE(REGTOOL_MODE_PENDINGEXP, pendingexpMode); + MODECASE(REGTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpMode); + MODECASE(REGTOOL_MODE_PROCESSEDEXP, processedexpMode); + MODECASE(REGTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool pendingimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit"); + return false; + } + + // select newImfiles that: + // exp_tag is in newExp + // don't have their exp_tag in rawExp + // XXX having the same exp_tag in newExp and raw*Exp is probably an error + // that should be checked for + + psString query = pxDataGet("regtool_pendingimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms) + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -simple"); + return false; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "regPendingImfile", !simple)) { + psError(PXTOOLS_ERR_PROG, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psString query = pxDataGet("regtool_find_unprocessed_imfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + { + // build a query to search by exp_tag, class, class_id + psMetadata *where = psMetadataAlloc(); + bool status = false; + psString exp_tag = psMetadataLookupStr(&status, config->args, "-exp_tag"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_tag"); + psFree(query); + return false; + } + if (exp_tag) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_tag", 0, "==", exp_tag)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_tag"); + psFree(where); + psFree(query); + return false; + } + } + + psString class_id = psMetadataLookupStr(&status, config->args, "-class_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id"); + psFree(query); + return false; + } + if (class_id) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "class_id", 0, "==", class_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item class_id"); + psFree(where); + psFree(query); + return false; + } + } + + // there's not + psString whereClaus = psDBGenerateWhereSQL(where, NULL); + psFree(where); + if (whereClaus) { + psStringAppend(&query, " %s", whereClaus); + psFree(whereClaus); + } + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, false, "no pending newImfile rows found"); + psFree(output); + return false; + } + + // insert 'newImfile's into rawImfile + if (psArrayLength(output) > 0) { + // start a transaction so we don't end up half of the imfiles we were + // trying to update uninserted + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + // convert newImfile metadata -> newImfile object + newImfileRow *object = newImfileObjectFromMetadata(output->data[i]); + // convert newImfile object -> rawImfile object + rawImfileRow *imfile = newToRawImfile(config, object); + if (!imfile) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to create a new rawImfile row"); + psFree(object); + psFree(output); + return false; + } + psFree(object); + // insert the rawImfile object into the database + if (!rawImfileInsertObject(config->dbh, imfile)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to insert row into the database"); + psFree(imfile); + psFree(output); + return false; + } + psFree(imfile); + } + + // point of no return for rawImfile + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = pxDataGet("regtool_processedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND rawImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND rawImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool updateprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + if (code == INT16_MAX) { + psError(PS_ERR_UNKNOWN, true, "-code is required"); + return false; + } + + if (!pxSetFaultCode(config->dbh, "rawImfile", config->where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + return false; + } + + return true; +} + + +static bool pendingexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit"); + return false; + } + + // return only exps that: + // are not in rawExp + // have ALL of their imfiles in rawImfile (by count) + // and have no associated imfiles left in newImfile + psString query = pxDataGet("regtool_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms) + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -simple"); + return false; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "regPendingExp", !simple)) { + psError(PXTOOLS_ERR_PROG, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // make sure that the exp_tag(s) are ready to be updated based on: + // exp_tag is not in rawExp + // exp_tag is not in newImfile + // that the correct count of imfiles is in rawImfile + + bool status = false; + psString exp_tag = psMetadataLookupStr(&status, config->args, "-exp_tag"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_tag"); + return false; + } + if (!exp_tag) { + psError(PS_ERR_UNKNOWN, true, "-exp_tag is required"); + return false; + } + + bool detrend = psMetadataLookupBool(&status, config->args, "-detrend"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -detrend"); + return false; + } + + psString query = pxDataGet("regtool_find_unprocessed_exp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query, exp_tag)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, false, "no pending newExp rows found"); + psFree(output); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // insert the exp into rawExp + for (long i = 0; psArrayLength(output) > i; i++) { + psMetadata *row = output->data[i]; + // convert metadata into a newExp object + newExpRow *newExp = newExpObjectFromMetadata(row); + // convert newExp object into a rawExp object + rawExpRow *rawExp = newToRawExp(config, newExp); + if (!rawExp) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to convert newExp to rawExp"); + psFree(newExp); + psFree(output); + return false; + } + psFree(newExp); + + // insert the rawExp object into the database + if (!rawExpInsertObject(config->dbh, rawExp)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(rawExp); + psFree(output); + return false; + } + psFree(rawExp); + + // if this is a detrend image don't put it in the chip queue (and we're + // done) + if (detrend) { + // set the state for the newExp to stop + if (!pxnewExpSetState(config, exp_tag, "stop")) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to change newExp.state for exp_tag: %s", exp_tag); + psFree(output); + return false; + } + continue; + } + + + // set the state for the newExp to stop + if (!pxnewExpSetState(config, exp_tag, "stop")) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to change newExp.state for exp_tag: %s", exp_tag); + psFree(output); + return false; + } + + // insert an entry into the chipPendingExp table + // this can only be run as the newExp's state has been set to stop + if (!pxchipQueueByExpTag(config, exp_tag, NULL, NULL, NULL, NULL, NULL)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to queue chipPendingExp"); + psFree(output); + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit"); + return false; + } + + bool faulted = psMetadataLookupU64(&status, config->args, "-faulted"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -faulted"); + return false; + } + + psString query = pxDataGet("regtool_processedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND rawExp.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND rawExp.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool updateprocessedexpMode(pxConfig *config) +{ + bool status = false; + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + if (code == INT16_MAX) { + psError(PS_ERR_UNKNOWN, true, "-code is required"); + return false; + } + + if (!pxSetFaultCode(config->dbh, "rawExp", config->where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + return false; + } + + return true; +} + + +# if 0 +static guidePendingExpRow *newToP1PendingExp(newExpRow *newExp) +{ + return guidePendingExpRowAlloc( + newExp->exp_tag, + newExp->camera, + newExp->telescope, + newExp->exp_type, + newExp->imfiles, + "my filter", + 0.1, // airmass + 0.2, // ra + 0.3, // dec + 0.4, // exp time + 0.5, // background + "my recipe", + 0xff // XXX calc version number + ); +} +#endif + + +#if 0 +static chipPendingExpRow *newTochipPendingExp(pxConfig *config, newExpRow *exp) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(exp, NULL); + + // optional + bool status = false; + psString label = psMetadataLookupStr(&status, config->args, "-label"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label"); + return false; + } + + chipPendingExpRow *chipExp = chipPendingExpRowAlloc( + 0, // chip ID, assigned by db + exp->exp_tag, + 0xdeadbeef, // guide version + label, + "my recipe", // recipe + NULL, // expgroup + NULL // dvodb + ); + + return chipExp; +} + + +static chipInputImfileRow *rawImfileTochipInputImfile(pxConfig *config, psS64 chip_id, rawImfileRow *rawImfile) +{ + return chipInputImfileRowAlloc( + chip_id, + rawImfile->class_id, + rawImfile->uri + ); +} +#endif + + +static rawExpRow *newToRawExp(pxConfig *config, newExpRow *exp) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(exp, NULL); + + bool status = false; + // optional + psString exp_type = psMetadataLookupStr(&status, config->args, "-exp_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_type"); + return false; + } + + psString filelevel = psMetadataLookupStr(&status, config->args, "-filelevel"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filelevel"); + return false; + } + if (!filelevel) { + psError(PS_ERR_UNKNOWN, true, "-filelevel is required"); + return false; + } + + psString filter = psMetadataLookupStr(&status, config->args, "-filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter"); + return false; + } + + psF32 airmass = psMetadataLookupF32(&status, config->args, "-airmass"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass"); + return false; + } + + psF64 ra = psMetadataLookupF64(&status, config->args, "-ra"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ra"); + return false; + } + + psF64 decl = psMetadataLookupF64(&status, config->args, "-decl"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -decl"); + return false; + } + + psF32 exp_time = psMetadataLookupF32(&status, config->args, "-exp_time"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time"); + return false; + } + + psF32 sat_pixel_frac = psMetadataLookupF32(&status, config->args, "-sat_pixel_frac"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -sat_pixel_frac"); + return false; + } + + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + + psF64 alt = psMetadataLookupF64(&status, config->args, "-alt"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -alt"); + return false; + } + + psF64 az = psMetadataLookupF64(&status, config->args, "-az"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -az"); + return false; + } + + psF32 ccd_temp = psMetadataLookupF32(&status, config->args, "-ccd_temp"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp"); + return false; + } + + psF64 posang = psMetadataLookupF32(&status, config->args, "-posang"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang"); + return false; + } + + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + psString object = psMetadataLookupStr(&status, config->args, "-object"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -object"); + return false; + } + + psTime *dateobs = NULL; + { + psString dateobsStr = psMetadataLookupStr(&status, config->args, "-dateobs"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -dateobs"); + return false; + } + // pass through NULL as this is an optional field + if (dateobsStr) { + dateobs = psTimeFromISO(dateobsStr, PS_TIME_UTC); + } else { + dateobs = NULL; + } + } + + // default + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + rawExpRow *raw = rawExpRowAlloc( + exp->exp_tag, + exp->exp_id, + exp->camera, + exp->telescope, + dateobs, + exp_type, + exp->imfiles, + filelevel, + exp->workdir, + filter, + airmass, + ra, + decl, + exp_time, + sat_pixel_frac, + bg, + bg_stdev, + bg_mean_stdev, + alt, + az, + ccd_temp, + posang, + user_1, + user_2, + user_3, + user_4, + user_5, + object, + 0.0, + code + ); + + if (dateobs) { + psFree(dateobs); + } + + return raw; +} + + +static rawImfileRow *newToRawImfile(pxConfig *config, newImfileRow *imfile) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(exp, NULL); + + bool status = false; + // optional + psString exp_type = psMetadataLookupStr(&status, config->args, "-exp_type"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_type"); + return false; + } + + psString filelevel = psMetadataLookupStr(&status, config->args, "-filelevel"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filelevel"); + return false; + } + + psString filter = psMetadataLookupStr(&status, config->args, "-filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter"); + return false; + } + + psF32 airmass = psMetadataLookupF32(&status, config->args, "-airmass"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass"); + return false; + } + + psF64 ra = psMetadataLookupF64(&status, config->args, "-ra"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ra"); + return false; + } + + psF64 decl = psMetadataLookupF64(&status, config->args, "-decl"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -decl"); + return false; + } + + psF32 exp_time = psMetadataLookupF32(&status, config->args, "-exp_time"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time"); + return false; + } + + psF32 sat_pixel_frac = psMetadataLookupF32(&status, config->args, "-sat_pixel_frac"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -sat_pixel_frac"); + return false; + } + + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + + psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev"); + return false; + } + + psF64 alt = psMetadataLookupF64(&status, config->args, "-alt"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -alt"); + return false; + } + + psF64 az = psMetadataLookupF64(&status, config->args, "-az"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -az"); + return false; + } + + psF32 ccd_temp = psMetadataLookupF32(&status, config->args, "-ccd_temp"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp"); + return false; + } + + psF64 posang = psMetadataLookupF32(&status, config->args, "-posang"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang"); + return false; + } + + psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1"); + return false; + } + psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2"); + return false; + } + psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3"); + return false; + } + psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4"); + return false; + } + psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5"); + return false; + } + + psString object = psMetadataLookupStr(&status, config->args, "-object"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -object"); + return false; + } + + psTime *dateobs = NULL; + { + psString dateobsStr = psMetadataLookupStr(&status, config->args, "-dateobs"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -dateobs"); + return false; + } + // pass through NULL as this is an optional field + if (dateobsStr) { + dateobs = psTimeFromISO(dateobsStr, PS_TIME_UTC); + } else { + dateobs = NULL; + } + } + + psS16 code = psMetadataLookupS16(&status, config->args, "-code"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code"); + return false; + } + + rawImfileRow *raw = rawImfileRowAlloc( + imfile->exp_tag, + imfile->class_id, + imfile->uri, + exp_type, + filelevel, + filter, + airmass, + ra, + decl, + exp_time, + sat_pixel_frac, + bg, + bg_stdev, + bg_mean_stdev, + alt, + az, + ccd_temp, + posang, + user_1, + user_2, + user_3, + user_4, + user_5, + object, + dateobs, + code + ); + + psFree(dateobs); + + return raw; +} Index: /branches/jh_branches/backtrack/ippTools/src/regtool.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/regtool.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/regtool.h (revision 22250) @@ -0,0 +1,39 @@ +/* + * regtool.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef REGTOOL_H +#define REGTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + REGTOOL_MODE_NONE = 0x0, + REGTOOL_MODE_PENDINGIMFILE, + REGTOOL_MODE_ADDPROCESSEDIMFILE, + REGTOOL_MODE_PROCESSEDIMFILE, + REGTOOL_MODE_UPDATEPROCESSEDIMFILE, + REGTOOL_MODE_PENDINGEXP, + REGTOOL_MODE_ADDPROCESSEDEXP, + REGTOOL_MODE_PROCESSEDEXP, + REGTOOL_MODE_UPDATEPROCESSEDEXP, +} regtoolMode; + +pxConfig *regtoolConfig(pxConfig *config, int argc, char **argv); + +#endif // REGTOOL_H Index: /branches/jh_branches/backtrack/ippTools/src/regtoolConfig.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/regtoolConfig.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/regtoolConfig.c (revision 22250) @@ -0,0 +1,321 @@ +/* + * regtoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include + +#include "pxtools.h" +#include "regtool.h" + +pxConfig *regtoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PXTOOLS_ERR_CONFIG, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -pendingimfile + psMetadata *pendingimfileArgs = psMetadataAlloc(); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search by exposure ID", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class", 0, + "search by class", NULL); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search by class ID", NULL); + psMetadataAddU64(pendingimfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addprocessedimfile + psMetadata *addprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-exp_tag", 0, + "define exposure ID (required)", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "define class ID (required)", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-exp_type", 0, + "define exposure type", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-filelevel", 0, + "define filelevel", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-filter", 0, + "define filter ", NULL); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-airmass", 0, + "define airmass", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-ra", 0, + "define RA", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-decl", 0, + "define DEC", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-exp_time", 0, + "define exposure time", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-sat_pixel_frac", 0, + "define fraction of saturated pixels", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-bg", 0, + "define exposue background", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposue background stdev", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposue background mean stdev", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-alt", 0, + "define altitute", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-az", 0, + "define azimuth", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-ccd_temp", 0, + "define ccd tempature", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-posang", 0, + "define rotator position angle", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_1", 0, + "define user statistic (1)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_2", 0, + "define user statistic (2)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_3", 0, + "define user statistic (3)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_4", 0, + "define user statistic (4)", NAN); + psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-user_5", 0, + "define user statistic (5)", NAN); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-object", 0, + "define exposure object", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-dateobs", 0, + "define observation time", NULL); + psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + + // -processedimfile + psMetadata *processedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search by exposure ID", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search by class ID", NULL); + psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -updateprocessedimfile + psMetadata *updateprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search by exposure ID", NULL); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, + "search by class ID", NULL); + psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, + "set fault code (required)", INT16_MAX); + + + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search by exposure ID", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-inst", 0, + "search by camera", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-telescope", 0, + "search by telescope", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-exp_type", 0, + "search by exposure type", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-imfiles", 0, + "search for exps with N imfiles", NULL); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addprocessedexp + psMetadata *addprocessedexpArgs = psMetadataAlloc(); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-exp_tag", 0, + "exp_tag to operate on (required)", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-exp_type", 0, + "define exposure type", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-filelevel", 0, + "define the data partitioning level of this file", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-workdir", 0, + "define the \"default\" workdir for this exposure", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-filter", 0, + "define filter ", NULL); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-airmass", 0, + "define airmass", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-ra", 0, + "define RA", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-decl", 0, + "define DEC", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-exp_time", 0, + "define exposure time", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-sat_pixel_frac", 0, + "define fraction of saturated pixels", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg", 0, + "define exposue background", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposue background stdev", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-bg_mean_stdev", 0, + "define exposue background mean stdev", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-alt", 0, + "define altitute", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-az", 0, + "define azimuth", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-ccd_temp", 0, + "define ccd tempature", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-posang", 0, + "define rotator position angle", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_1", 0, + "define user statistic (1)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_2", 0, + "define user statistic (2)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_3", 0, + "define user statistic (3)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_4", 0, + "define user statistic (4)", NAN); + psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_5", 0, + "define user statistic (5)", NAN); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-object", 0, + "define exposure object", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-dateobs", 0, + "define observation time", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-label", 0, + "define label for phase 2 processing (non-detrend data only)", NULL); + psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-code", 0, + "set fault code", 0); + /* + psMetadataAddStr(updateArgs, PS_LIST_TAIL, "-recip", 0, + "define URL", NULL); + psMetadataAddStr(updateArgs, PS_LIST_TAIL, "-mosiac", 0, + "define URL", NULL); + */ + psMetadataAddBool(addprocessedexpArgs, PS_LIST_TAIL, "-detrend", 0, + "declare this as detrend data", false); + + // -processedexp + psMetadata *processedexpArgs = psMetadataAlloc(); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search by exposure ID", NULL); + psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, + "only return imfiles with a fault status set", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -updatedprocessedexp + psMetadata *updatedprocessedexpArgs = psMetadataAlloc(); + psMetadataAddStr(updatedprocessedexpArgs, PS_LIST_TAIL, "-exp_tag", 0, + "search by exposure ID", NULL); + psMetadataAddS16(updatedprocessedexpArgs, PS_LIST_TAIL, "-code", 0, + "set fault code (required)", INT16_MAX); + + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXTOOL_ADD_MODE("-pendingimfile", "", REGTOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + PXTOOL_ADD_MODE("-addprocessedimfile", "", REGTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); + PXTOOL_ADD_MODE("-processedimfile", "", REGTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); + PXTOOL_ADD_MODE("-updateprocessedimfile", "", REGTOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileArgs); + PXTOOL_ADD_MODE("-pendingexp", "", REGTOOL_MODE_PENDINGEXP,pendingexpArgs); + PXTOOL_ADD_MODE("-addprocessedexp", "", REGTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs); + PXTOOL_ADD_MODE("-processedexp", "", REGTOOL_MODE_PROCESSEDEXP, processedexpArgs); + PXTOOL_ADD_MODE("-updateprocessedexp", "", REGTOOL_MODE_UPDATEPROCESSEDEXP, updatedprocessedexpArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // setup search criterion +#define addWhereStr(name) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, #name, 0, "==", str)) {\ + psError(PXTOOLS_ERR_PROG, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + + // generate SQL where clause + config->where = psMetadataAlloc(); + + addWhereStr(exp_tag); + // convert '-inst' to 'camera' + { + psString str = NULL; + bool status = false; + if ((str = psMetadataLookupStr(&status, config->args, "-inst"))) { + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, "camera", 0, "==", str)) { + psError(PXTOOLS_ERR_PROG, false, "failed to add item camera"); + psFree(config); + return NULL; + } + } + } + addWhereStr(telescope); + addWhereStr(exp_type); + { + int imfiles = 0; + bool status = false; + if ((imfiles = psMetadataLookupS32(&status, config->args, "-imfiles"))) { + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, "imfiles", 0, "==", imfiles)) { + psError(PXTOOLS_ERR_PROG, false, "failed to add item imfiles"); + psFree(config); + return NULL; + } + } + } + addWhereStr(class); + addWhereStr(class_id); + addWhereStr(filter); + + if (config->where->list->n < 1) { + psFree(config->where); + config->where = NULL; + } + + + // define Database handle, if used + config->dbh = pmConfigDB(config->modules); + if (!config->dbh) { + psError(PXTOOLS_ERR_SYS, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/jh_branches/backtrack/ippTools/src/stacktool.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/stacktool.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/stacktool.c (revision 22250) @@ -0,0 +1,685 @@ +/* + * stacktool.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "stacktool.h" + +static bool definerunMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool addinputskyfileMode(pxConfig *config); +static bool inputskyfileMode(pxConfig *config); +static bool tosumMode(pxConfig *config); +static bool addsumskyfileMode(pxConfig *config); +static bool sumskyfileMode(pxConfig *config); + +static bool setstackRunState(pxConfig *config, const char *stack_id, const char *state); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = stacktoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(STACKTOOL_MODE_DEFINERUN, definerunMode); + MODECASE(STACKTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(STACKTOOL_MODE_ADDINPUTSKYFILE, addinputskyfileMode); + MODECASE(STACKTOOL_MODE_INPUTSKYFILE, inputskyfileMode); + MODECASE(STACKTOOL_MODE_TOSUM, tosumMode); + MODECASE(STACKTOOL_MODE_ADDSUMSKYFILE, addsumskyfileMode); + MODECASE(STACKTOOL_MODE_SUMSKYFILE, sumskyfileMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + bool status = false; + psString workdir = psMetadataLookupStr(&status, config->args, "-workdir"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -workdir"); + return false; + } + if (!workdir) { + psError(PS_ERR_UNKNOWN, true, "-workdir is required"); + return false; + } + + psString skycell_id = psMetadataLookupStr(&status, config->args, "-skycell_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -skycell_id"); + return false; + } + if (!skycell_id) { + psError(PS_ERR_UNKNOWN, true, "-skycell_id is required"); + return false; + } + + psString tess_id = psMetadataLookupStr(&status, config->args, "-tess_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -tess_id"); + return false; + } + if (!tess_id) { + psError(PS_ERR_UNKNOWN, true, "-tess_id is required"); + return false; + } + + // we have to support multipe exp_tags + psMetadataItem *warp_ids = psMetadataLookup(config->args, "-warp_id"); + if (!warp_ids) { + // this shouldn't actually happen when using psArgs + psError(PS_ERR_UNKNOWN, true, "-warp_id is required"); + return false; + } + + psTime *registered = NULL; + { + psString registeredStr = psMetadataLookupStr(&status, config->args, "-registered"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -registered"); + return false; + } + // pass through NULL as this is an optional field + if (registeredStr) { + registered = psTimeFromISO(registeredStr, PS_TIME_UTC); + } else { + registered = NULL; + } + } + + stackRunRow *run = stackRunRowAlloc( + 0, // ID + "run", // state + workdir, + NULL, // dvodb + registered, + skycell_id, + tess_id + ); + psFree(registered); + if (!run) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc stackRun object"); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!stackRunInsertObject(config->dbh, run)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return true; + } + + // get the assigned warp_id + run->stack_id = psDBLastInsertID(config->dbh); + + // insert the stackInputSkyfile rows + psListIterator *iter = psListIteratorAlloc(warp_ids->data.list, 0, false); + psMetadataItem *item = NULL; + while ((item = psListGetAndIncrement(iter))) { + // if the value is NULL this is probably the first pass through the + // loop and -warp_id was not specified at all + if (!item->data.V) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -warp_id"); + return false; + } + if (!stackInputSkyfileInsert(config->dbh, run->stack_id, (psS64)atoll((char *)item->data.V))) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to insert stackInputSkyfile rows"); + return false; + } + } + psFree(iter); + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(run); + return false; + } + } + + if (!stackRunPrintObject(stdout, run, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(run); + return false; + } + + psFree(run); + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString stack_id = psMetadataLookupStr(&status, config->args, "-stack_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -stack_id"); + return false; + } + if (!stack_id) { + psError(PS_ERR_UNKNOWN, true, "-stack_id is required"); + return false; + } + + psString state = psMetadataLookupStr(&status, config->args, "-state"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -state"); + return false; + } + if (!state) { + psError(PS_ERR_UNKNOWN, true, "-state is required"); + return false; + } + + if (state) { + // set detRun.state to state + return setstackRunState(config, stack_id, state); + } + + return true; +} + + +static bool addinputskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString stack_id = psMetadataLookupStr(&status, config->args, "-stack_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -stack_id"); + return false; + } + if (!stack_id) { + psError(PS_ERR_UNKNOWN, true, "-stack_id is required"); + return false; + + } + + psString warp_id = psMetadataLookupStr(&status, config->args, "-warp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -warp_id"); + return false; + } + if (!warp_id) { + psError(PS_ERR_UNKNOWN, true, "-warp_id is required"); + return false; + } + + + // XXX need to validate the warp_id here + // XXX instead of validiting it here we should just use forgein key + // constrants + if (!stackInputSkyfileInsert(config->dbh, + (psS64)atoll(stack_id), + (psS64)atoll(warp_id) + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool inputskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("stacktool_inputskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "stackInputSkyfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (psArrayLength(output)) { + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackInputSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool tosumMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("stacktool_tosum.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "stackSumSkyfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (psArrayLength(output)) { + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackSumSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool addsumskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString stack_id = psMetadataLookupStr(&status, config->args, "-stack_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -stack_id"); + return false; + } + if (!stack_id) { + psError(PS_ERR_UNKNOWN, true, "-stack_id is required"); + return false; + + } + + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + if (!path_base) { + psError(PS_ERR_UNKNOWN, true, "-path_base is required"); + return false; + } + + // optional + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // XXX need to validate the stack_id here + // XXX instead of validiting it here we should just use forgein key + // constrants + if (!stackSumSkyfileInsert(config->dbh, + (psS64)atoll(stack_id), + uri, + path_base, + bg, + bg_stdev + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!setstackRunState(config, stack_id, "stop")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to change stackRun's state"); + return false; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool sumskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit"); + return false; + } + + psString query = pxDataGet("stacktool_sumskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "stackSumSkyfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (psArrayLength(output)) { + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + if (!ippdbPrintMetadatas(stdout, output, "stackSumSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool setstackRunState(pxConfig *config, const char *stack_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(stack_id, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "reg", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid warpRun state: %s", state); + return false; + } + + char *query = "UPDATE stackRun SET state = '%s' WHERE stack_id = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, state, stack_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for stack_id %s", stack_id); + return false; + } + + return true; +} Index: /branches/jh_branches/backtrack/ippTools/src/stacktool.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/stacktool.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/stacktool.h (revision 22250) @@ -0,0 +1,38 @@ +/* + * stacktool.h + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef STACKTOOL_H +#define STACKTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + STACKTOOL_MODE_NONE = 0x0, + STACKTOOL_MODE_DEFINERUN, + STACKTOOL_MODE_UPDATERUN, + STACKTOOL_MODE_ADDINPUTSKYFILE, + STACKTOOL_MODE_INPUTSKYFILE, + STACKTOOL_MODE_TOSUM, + STACKTOOL_MODE_ADDSUMSKYFILE, + STACKTOOL_MODE_SUMSKYFILE, +} stacktoolMode; + +pxConfig *stacktoolConfig(pxConfig *config, int argc, char **argv); + +#endif // STACKTOOL_H Index: /branches/jh_branches/backtrack/ippTools/src/stacktoolConfig.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/stacktoolConfig.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/stacktoolConfig.c (revision 22250) @@ -0,0 +1,218 @@ +/* + * stacktoolConfig.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "stacktool.h" + +pxConfig *stacktoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *time = psTimeGetNow(PS_TIME_TAI); + psString now = psTimeToISO(time); + psFree(time); + + // -definerun + psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, + "define workdir (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-registered", 0, + "time detrend run was registered", now); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-skycell_id", 0, + "define skycell ID (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, + "define tessellation ID (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-warp_id", + PS_META_DUPLICATE_OK, + "include this warp ID (multiple OK, required)", NULL); + + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-stack_id", 0, + "define stack ID (required)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, + "set state (required)", NULL); +#if 0 + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-workdir", 0, + "define workdir (required)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-registered", 0, + "time detrend run was registered", now); +#endif + + // -addinputskyfile + psMetadata *addinputskyfileArgs = psMetadataAlloc(); + psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, + "define stack ID (required)", NULL); + psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, + "define warp ID (required)", NULL); + + // -inputskyfile + psMetadata *inputskyfileArgs = psMetadataAlloc(); + psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, + "search by stack ID", NULL); + psMetadataAddStr(inputskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, + "search by warp ID", NULL); + psMetadataAddU64(inputskyfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -tosum + psMetadata *tosumArgs = psMetadataAlloc(); + psMetadataAddStr(tosumArgs, PS_LIST_TAIL, "-stack_id", 0, + "search by stack ID", NULL); + psMetadataAddStr(tosumArgs, PS_LIST_TAIL, "-warp_id", 0, + "search by warp ID", NULL); + psMetadataAddU64(tosumArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(tosumArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addsumskyfile + psMetadata *addsumskyfileArgs = psMetadataAlloc(); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, + "define warp ID (required)", NULL); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-uri", 0, + "define URI of file (required)", 0); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location (required)", 0); + psMetadataAddF64(addsumskyfileArgs, PS_LIST_TAIL, "-bg", 0, + "define exposue background", NAN); + psMetadataAddF64(addsumskyfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposue background mean stdev", NAN); + + // -sumskyfile + psMetadata *sumskyfileArgs= psMetadataAlloc(); + psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, + "search by stack ID", NULL); + psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, + "search by warp ID", NULL); + psMetadataAddU64(sumskyfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(sumskyfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXTOOL_ADD_MODE("-definerun", "", STACKTOOL_MODE_DEFINERUN, definerunArgs); + PXTOOL_ADD_MODE("-updaterun", "", STACKTOOL_MODE_UPDATERUN, updaterunArgs); + PXTOOL_ADD_MODE("-addinputskyfile", "", STACKTOOL_MODE_ADDINPUTSKYFILE, addinputskyfileArgs); + PXTOOL_ADD_MODE("-inputskyfile", "", STACKTOOL_MODE_INPUTSKYFILE, inputskyfileArgs); + PXTOOL_ADD_MODE("-tosum", "", STACKTOOL_MODE_TOSUM, tosumArgs); + PXTOOL_ADD_MODE("-addsumskyfile", "", STACKTOOL_MODE_ADDSUMSKYFILE, addsumskyfileArgs); + PXTOOL_ADD_MODE("-sumskyfile", "", STACKTOOL_MODE_SUMSKYFILE, sumskyfileArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // setup search criterion +#define addWhereStr(name) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, #name, 0, "==", str)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define addWhereS32(name) \ +{ \ + psS32 s32 = 0; \ + bool status = false; \ + if ((s32= psMetadataLookupS32(&status, config->args, "-" #name))) { \ + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, #name, 0, "==", s32)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + + + // generate SQL where clause + config->where = psMetadataAlloc(); + +{ + psString str = NULL; + bool status = false; + if ((str = psMetadataLookupStr(&status, config->args, "-warp_id"))) { + if (!psMetadataAddS64(config->where, PS_LIST_TAIL, "warp_id", 0, "==", (psS64)atoll(str))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item warp_id"); + psFree(config); + return NULL; + } + } +} + addWhereStr(skycell_id); + addWhereStr(tess_id); + addWhereStr(exp_tag); + + if (config->where->list->n < 1) { + psFree(config->where); + config->where = NULL; + } + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = pmConfigDB(config->modules); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/jh_branches/backtrack/ippTools/src/warptool.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/warptool.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/warptool.c (revision 22250) @@ -0,0 +1,1168 @@ +/* + * warptool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "warptool.h" + +static bool definerunMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool addinputexpMode(pxConfig *config); +static bool expMode(pxConfig *config); +static bool imfileMode(pxConfig *config); +static bool tooverlapMode(pxConfig *config); +static bool addoverlapMode(pxConfig *config); +static bool scmapMode(pxConfig *config); +static bool towarpedMode(pxConfig *config); +static bool addwarpedMode(pxConfig *config); +static bool warpedMode(pxConfig *config); + +static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile); +static bool setwarpRunState(pxConfig *config, const char *warp_id, const char *state); +static bool isValidMode(pxConfig *config, const char *mode); +bool warpCompletedRuns(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = warptoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(WARPTOOL_MODE_DEFINERUN, definerunMode); + MODECASE(WARPTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(WARPTOOL_MODE_ADDINPUTEXP, addinputexpMode); + MODECASE(WARPTOOL_MODE_EXP, expMode); + MODECASE(WARPTOOL_MODE_IMFILE, imfileMode); + MODECASE(WARPTOOL_MODE_TOOVERLAP, tooverlapMode); + MODECASE(WARPTOOL_MODE_ADDOVERLAP, addoverlapMode); + MODECASE(WARPTOOL_MODE_SCMAP, scmapMode); + MODECASE(WARPTOOL_MODE_TOWARPED, towarpedMode); + MODECASE(WARPTOOL_MODE_ADDWARPED, addwarpedMode); + MODECASE(WARPTOOL_MODE_WARPED, warpedMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + bool status = false; + psString mode = psMetadataLookupStr(&status, config->args, "-mode"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -survey_mode"); + return false; + } + if (!mode) { + psError(PS_ERR_UNKNOWN, true, "-mode is required"); + return false; + } + // check mode + if (mode && !isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalud mode"); + return false; + } + + psString workdir = psMetadataLookupStr(&status, config->args, "-workdir"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -workdir"); + return false; + } + if (!workdir) { + psError(PS_ERR_UNKNOWN, true, "-workdir is required"); + return false; + } + + psTime *registered = NULL; + { + psString registeredStr = psMetadataLookupStr(&status, config->args, "-registered"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -registered"); + return false; + } + // pass through NULL as this is an optional field + if (registeredStr) { + registered = psTimeFromISO(registeredStr, PS_TIME_UTC); + } else { + registered = NULL; + } + } + + warpRunRow *warpRun = warpRunRowAlloc( + 0, // ID + mode, + "reg", // state + workdir, + NULL, // dvodb + registered + ); + psFree(registered); + if (!warpRun) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc warpRun object"); + return true; + } + if (!warpRunInsertObject(config->dbh, warpRun)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpRun); + return true; + } + + // get the assigned warp_id + warpRun->warp_id = psDBLastInsertID(config->dbh); + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + psFree(warpRun); + return false; + } + } + + if (!warpRunPrintObject(stdout, warpRun, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(warpRun); + return false; + } + + psFree(warpRun); + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString warp_id = psMetadataLookupStr(&status, config->args, "-warp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -warpt_id"); + return false; + } + if (!warp_id) { + psError(PS_ERR_UNKNOWN, true, "-warp_id is required"); + return false; + } + + psString state = psMetadataLookupStr(&status, config->args, "-state"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -state"); + return false; + } + if (!state) { + psError(PS_ERR_UNKNOWN, true, "-state is required"); + return false; + } + + if (state) { + // set detRun.state to state + return setwarpRunState(config, warp_id, state); + } + + return true; +} + + +static bool addinputexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString warp_id = psMetadataLookupStr(&status, config->args, "-warp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -warp_id"); + return false; + } + if (!warp_id) { + psError(PS_ERR_UNKNOWN, true, "-warp_id is required"); + return false; + } + + psString cam_id = psMetadataLookupStr(&status, config->args, "-cam_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -cam_id"); + return false; + } + if (!cam_id) { + psError(PS_ERR_UNKNOWN, true, "-cam_id is required"); + return false; + } + + // defaults to false + bool magiced = psMetadataLookupBool(&status, config->args, "-magiced"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -magiced"); + return false; + } + + // XXX need to validate the warp_id here + // XXX instead of validiting it here we should just use forgein key + // constrants + if (!warpInputExpInsert(config->dbh, + (psS64)atoll(warp_id), + (psS64)atoll(cam_id), + magiced + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool expMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit"); + return false; + } + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_exp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "warpInputExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (psArrayLength(output)) { + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpInputExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool imfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit"); + return false; + } + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_imfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "warpInputExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (psArrayLength(output)) { + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpInputImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool tooverlapMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit"); + return false; + } + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_tooverlap.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "warpInputExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (psArrayLength(output)) { + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpInputExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool addoverlapMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString mapfile = psMetadataLookupStr(&status, config->args, "-mapfile"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -mapfile"); + return false; + } + if (!mapfile) { + psError(PS_ERR_UNKNOWN, true, "-mapfile is required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!parseAndInsertSkyCellMap(config, mapfile)) { + psError(PS_ERR_UNKNOWN, false, "failed to inject mapfile: %s into the database", mapfile); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile) +{ + unsigned int nFail = 0; + psMetadata *skycells = psMetadataConfigRead(NULL, &nFail, mapfile, false); + if (!skycells) { + psError(PS_ERR_UNKNOWN, false, "failed to parse mapfile: %s", mapfile); + return false; + } + if (nFail) { + psError(PS_ERR_UNKNOWN, false, "there were %d errors parsing mapfile: %s", nFail, mapfile); + psFree(skycells); + return false; + } + + psMetadataItem *item = NULL; + psMetadataIterator *iter = psMetadataIteratorAlloc(skycells, 0, NULL); + while ((item = psMetadataGetAndIncrement(iter))) { + if (item->type != PS_DATA_METADATA) { + psError(PS_ERR_UNKNOWN, false, "mapfile: %s is in the wrong format", mapfile); + psFree(iter); + psFree(skycells); + return false; + } + + psMetadata *sc = item->data.md; + // this conversion isn't strictly nessicary but it's an easy way of + // validating the format + warpSkyCellMapRow *row = warpSkyCellMapObjectFromMetadata(sc); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert mapfile: %s metdata entry into a warpSkyCellMap object", mapfile); + psFree(iter); + psFree(skycells); + return false; + } + + if (!warpSkyCellMapInsertObject(config->dbh, row)) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + psFree(row); + psFree(iter); + psFree(skycells); + return false; + } + + psFree(row); + } + psFree(iter); + psFree(skycells); + + return true; +} + + +static bool scmapMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit"); + return false; + } + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_scmap.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "warpSkyCellMap"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (psArrayLength(output)) { + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpSkyCellMap", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool towarpedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit"); + return false; + } + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_towarped.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "warpSkyCellMap"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (psArrayLength(output)) { + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpPendingSkyCell", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool addwarpedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psString warp_id = psMetadataLookupStr(&status, config->args, "-warp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -warp_id"); + return false; + } + if (!warp_id) { + psError(PS_ERR_UNKNOWN, true, "-warp_id is required"); + return false; + } + + psString skycell_id = psMetadataLookupStr(&status, config->args, "-skycell_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -skycell_id"); + return false; + } + if (!skycell_id) { + psError(PS_ERR_UNKNOWN, true, "-skycell_id is required"); + return false; + } + + psString tess_id = psMetadataLookupStr(&status, config->args, "-tess_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -tess_id"); + return false; + } + if (!tess_id) { + psError(PS_ERR_UNKNOWN, true, "-tess_id is required"); + return false; + } + + psString uri = psMetadataLookupStr(&status, config->args, "-uri"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -uri"); + return false; + } + if (!uri) { + psError(PS_ERR_UNKNOWN, true, "-uri is required"); + return false; + } + + psString path_base = psMetadataLookupStr(&status, config->args, "-path_base"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -path_base"); + return false; + } + if (!path_base) { + psError(PS_ERR_UNKNOWN, true, "-path_base is required"); + return false; + } + + // optional + psF64 bg = psMetadataLookupF64(&status, config->args, "-bg"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg"); + return false; + } + + psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev"); + return false; + } + + // we don't want to insert the last skyfile in a run but then not mark the + // run as 'stop' + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // XXX need to validate that this coresponds to an warpInputImfile + if (!warpSkyfileInsert(config->dbh, + (psS64)atoll(warp_id), + skycell_id, + tess_id, + uri, + path_base, + bg, + bg_stdev + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!warpCompletedRuns(config)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +bool warpCompletedRuns(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // XXX this SQL has not been broken out to into seperate files as the MYSQL + // < 5 & MYSQL 5 versions need to be kept in sync +#if MYSQL5 + // XXX at MySQL 4.1.21 (probably all of 4.1.x) chokes and dies on this + // statement as it thinks it is trying to select from the table being + // updated. The 4.1 manual says that nested sub-queries are explicited + // allowed to do this with update statements as a temporary table is + // created so that you are not actually selecting from the table you are + // modifying. + char *query = + "UPDATE warpRun\n" + " SET warpRun.state = 'stop'\n" + " WHERE\n" + " warpRun.warp_id =\n" + " (SELECT DISTINCT\n" + " warp_id\n" + " FROM\n" + " (SELECT DISTINCT\n" + " warpRun.warp_id,\n" + " warpSkyCellMap.warp_id as foo,\n" + " warpSkyfile.warp_id as bar\n" + " FROM warpRun\n" + " JOIN warpSkyCellMap\n" + " USING(warp_id)\n" + " LEFT JOIN warpSkyfile\n" + " USING(warp_id, skycell_id, tess_id)\n" + " WHERE\n" + " warpRun.state = 'run'\n" + " GROUP BY\n" + " warpRun.warp_id\n" + " HAVING\n" + " COUNT(warpSkyCellMap.warp_id) = COUNT(warpSkyfile.warp_id)\n" + " ) as Foo\n" + " )\n"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } +#else // if MYSQL5 +{ + char *query = + "CREATE TEMPORARY TABLE finished\n" + " (warp_id INT, PRIMARY KEY(warp_id)) ENGINE=MEMORY\n"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } +} + +{ + char *query = + "INSERT INTO finished\n" + " SELECT\n" + " warp_id\n" + " FROM\n" + " (SELECT DISTINCT\n" + " warpRun.warp_id,\n" + " warpSkyCellMap.warp_id as foo,\n" + " warpSkyfile.warp_id as bar\n" + " FROM warpRun\n" + " JOIN warpSkyCellMap\n" + " USING(warp_id)\n" + " LEFT JOIN warpSkyfile\n" + " USING(warp_id, skycell_id, tess_id)\n" + " WHERE\n" + " warpRun.state = 'run'\n" + " GROUP BY\n" + " warpRun.warp_id\n" + " HAVING\n" + " COUNT(warpSkyCellMap.warp_id) = COUNT(warpSkyfile.warp_id)\n" + " ) as Foo \n"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } +} + +{ + char *query = + "UPDATE warpRun\n" + " SET warpRun.state = 'stop'\n" + " WHERE\n" + " warpRun.warp_id =\n" + " (SELECT DISTINCT\n" + " warp_id\n" + " FROM finished\n" + " )\n"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } +} +#endif // if MYSQL5 + + return true; +} + +static bool warpedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + psU64 limit = psMetadataLookupU64(&status, config->args, "-limit"); + if (!status) { + psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit"); + return false; + } + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_warped.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, "warpSkyfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + bool simple = false; + { + bool status = false; + simple = psMetadataLookupBool(&status, config->args, "-simple"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); + return false; + } + } + + if (psArrayLength(output)) { + if (!convertIdToStr(output)) { + psError(PS_ERR_UNKNOWN, false, "failed to convert id fields into a strings"); + psFree(output); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool setwarpRunState(pxConfig *config, const char *warp_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(warp_id, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "run", 4) == 0) + || (strncmp(state, "stop", 5) == 0) + || (strncmp(state, "reg", 4) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid warpRun state: %s", state); + return false; + } + + char *query = "UPDATE warpRun SET state = '%s' WHERE warp_id = '%s'"; + if (!p_psDBRunQuery(config->dbh, query, state, warp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for warp_id %s", warp_id); + return false; + } + + return true; +} + + +static bool isValidMode(pxConfig *config, const char *mode) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(mode, false); + + // check that state is a valid string value + if (!( + (strncmp(mode, "warp", 5) == 0) + || (strncmp(mode, "diff", 5) == 0) + || (strncmp(mode, "stack", 6) == 0) + || (strncmp(mode, "magic", 6) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid warpRun mode: %s", mode); + return false; + } + + return true; +} Index: /branches/jh_branches/backtrack/ippTools/src/warptool.h =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/warptool.h (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/warptool.h (revision 22250) @@ -0,0 +1,42 @@ +/* + * warptool.h + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef WARPTOOL_H +#define WARPTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + WARPTOOL_MODE_NONE = 0x0, + WARPTOOL_MODE_DEFINERUN, + WARPTOOL_MODE_UPDATERUN, + WARPTOOL_MODE_ADDINPUTEXP, + WARPTOOL_MODE_EXP, + WARPTOOL_MODE_IMFILE, + WARPTOOL_MODE_TOOVERLAP, + WARPTOOL_MODE_ADDOVERLAP, + WARPTOOL_MODE_SCMAP, + WARPTOOL_MODE_TOWARPED, + WARPTOOL_MODE_ADDWARPED, + WARPTOOL_MODE_WARPED, +} warptoolMode; + +pxConfig *warptoolConfig(pxConfig *config, int argc, char **argv); + +#endif // WARPTOOL_H Index: /branches/jh_branches/backtrack/ippTools/src/warptoolConfig.c =================================================================== --- /branches/jh_branches/backtrack/ippTools/src/warptoolConfig.c (revision 22250) +++ /branches/jh_branches/backtrack/ippTools/src/warptoolConfig.c (revision 22250) @@ -0,0 +1,274 @@ +/* + * warptoolConfig.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "warptool.h" + +pxConfig *warptoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *time = psTimeGetNow(PS_TIME_TAI); + psString now = psTimeToISO(time); + psFree(time); + + // -definerun + psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-mode", 0, + "define mode (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, + "define workdir (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-registered", 0, + "time detrend run was registered", now); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-warp_id", 0, + "define warptool ID (required)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, + "set state (required)", NULL); +#if 0 + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-workdir", 0, + "define workdir (required)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-registered", 0, + "time detrend run was registered", now); +#endif + + // -addinputexp + psMetadata *addinputexpArgs = psMetadataAlloc(); + psMetadataAddStr(addinputexpArgs, PS_LIST_TAIL, "-warp_id", 0, + "define warptool ID (required)", NULL); + psMetadataAddStr(addinputexpArgs, PS_LIST_TAIL, "-cam_id", 0, + "define camtool ID (required)", NULL); + psMetadataAddBool(addinputexpArgs, PS_LIST_TAIL, "-magiced", 0, + "has this exposure been magiced", false); + + // -exp + psMetadata *expArgs = psMetadataAlloc(); + psMetadataAddStr(expArgs, PS_LIST_TAIL, "-warp_id", 0, + "search by warptool ID", NULL); + psMetadataAddStr(expArgs, PS_LIST_TAIL, "-cam_id", 0, + "search by camtool ID", NULL); + psMetadataAddU64(expArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(expArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -imfile + psMetadata *imfileArgs = psMetadataAlloc(); + psMetadataAddStr(imfileArgs, PS_LIST_TAIL, "-warp_id", 0, + "search by warptool ID", NULL); + psMetadataAddStr(imfileArgs, PS_LIST_TAIL, "-cam_id", 0, + "search by camtool ID", NULL); + psMetadataAddU64(imfileArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(imfileArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -tooverlap + psMetadata *tooverlapArgs = psMetadataAlloc(); + psMetadataAddStr(tooverlapArgs, PS_LIST_TAIL, "-warp_id", 0, + "search by warp ID", NULL); + psMetadataAddU64(tooverlapArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(tooverlapArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addoverlap + psMetadata *addoverlapArgs = psMetadataAlloc(); + psMetadataAddStr(addoverlapArgs, PS_LIST_TAIL, "-mapfile", 0, + "path to skycell <-> imfile mapping file", NULL); + + // -scmap + psMetadata *scmapArgs = psMetadataAlloc(); + psMetadataAddStr(scmapArgs, PS_LIST_TAIL, "-warp_id", 0, + "search by warptool ID", NULL); + psMetadataAddStr(scmapArgs, PS_LIST_TAIL, "-skycell_id", 0, + "searcy by skycell ID", NULL); + psMetadataAddStr(scmapArgs, PS_LIST_TAIL, "-tess_id", 0, + "searcy by tess ID", NULL); + psMetadataAddU64(scmapArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(scmapArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -towarped + psMetadata *towarpedArgs = psMetadataAlloc(); + psMetadataAddStr(towarpedArgs, PS_LIST_TAIL, "-warp_id", 0, + "search by warptool ID", NULL); + psMetadataAddU64(towarpedArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(towarpedArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + // -addwarped + psMetadata *addwarpedArgs = psMetadataAlloc(); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-warp_id", 0, + "define warptool ID (required)", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-skycell_id", 0, + "define skycell ID (required)", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-tess_id", 0, + "define tessellation ID (required)", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-uri", 0, + "define URI of file (required)", 0); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-path_base", 0, + "define base output location (required)", 0); + psMetadataAddF64(addwarpedArgs, PS_LIST_TAIL, "-bg", 0, + "define exposure background", NAN); + psMetadataAddF64(addwarpedArgs, PS_LIST_TAIL, "-bg_stdev", 0, + "define exposure background stdev", NAN); + + // -warped + psMetadata *warpedArgs = psMetadataAlloc(); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-warp_id", 0, + "search by warptool ID", NULL); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-skycell_id", 0, + "define skycell ID", NULL); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-tess_id", 0, + "define tessellation ID", NULL); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-exp_tag", 0, + "define exposure tag", NULL); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-cam_id", 0, + "define phase 3 version of exposure tag", NULL); + psMetadataAddU64(warpedArgs, PS_LIST_TAIL, "-limit", 0, + "limit result set to N items", 0); + psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-simple", 0, + "use the simple output format", false); + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXTOOL_ADD_MODE("-definerun", "", WARPTOOL_MODE_DEFINERUN, definerunArgs); + PXTOOL_ADD_MODE("-updaterun", "", WARPTOOL_MODE_UPDATERUN, updaterunArgs); + PXTOOL_ADD_MODE("-addinputexp", "", WARPTOOL_MODE_ADDINPUTEXP, addinputexpArgs); + PXTOOL_ADD_MODE("-exp", "", WARPTOOL_MODE_EXP, expArgs); + PXTOOL_ADD_MODE("-imfile", "", WARPTOOL_MODE_IMFILE, imfileArgs); + PXTOOL_ADD_MODE("-tooverlap", "", WARPTOOL_MODE_TOOVERLAP, tooverlapArgs); + PXTOOL_ADD_MODE("-addoverlap", "", WARPTOOL_MODE_ADDOVERLAP, addoverlapArgs); + PXTOOL_ADD_MODE("-scmap", "", WARPTOOL_MODE_SCMAP, scmapArgs); + PXTOOL_ADD_MODE("-towarped", "", WARPTOOL_MODE_TOWARPED, towarpedArgs); + PXTOOL_ADD_MODE("-addwarped", "", WARPTOOL_MODE_ADDWARPED, addwarpedArgs); + PXTOOL_ADD_MODE("-warped", "", WARPTOOL_MODE_WARPED, warpedArgs); +#if 0 + PXTOOL_ADD_MODE("-tostackedimfile", WARPTOOL_MODE_TOSTACKEDIMFILE, tostackedimfileArgs); + PXTOOL_ADD_MODE("-addstackedimfile", WARPTOOL_MODE_ADDSTACKEDIMFILE, addstackedimfileArgs); + PXTOOL_ADD_MODE("-stackedimfile", WARPTOOL_MODE_STACKEDIMFILE, stackedimfileArgs); + PXTOOL_ADD_MODE("-todiffimfile", WARPTOOL_MODE_TODIFFIMFILE, todiffimfileArgs); + PXTOOL_ADD_MODE("-adddiffimfile", WARPTOOL_MODE_ADDDIFFIMFILE, adddiffimfileArgs); + PXTOOL_ADD_MODE("-diffimfile", WARPTOOL_MODE_DIFFIMFILE, diffimfileArgs); +#endif + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // setup search criterion +#define addWhereStr(name) \ +{ \ + psString str = NULL; \ + bool status = false; \ + if ((str = psMetadataLookupStr(&status, config->args, "-" #name))) { \ + if (!psMetadataAddStr(config->where, PS_LIST_TAIL, #name, 0, "==", str)) {\ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + +#define addWhereS32(name) \ +{ \ + psS32 s32 = 0; \ + bool status = false; \ + if ((s32= psMetadataLookupS32(&status, config->args, "-" #name))) { \ + if (!psMetadataAddS32(config->where, PS_LIST_TAIL, #name, 0, "==", s32)) { \ + psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \ + psFree(config); \ + return NULL; \ + } \ + } \ +} + + + // generate SQL where clause + config->where = psMetadataAlloc(); + +{ + psString str = NULL; + bool status = false; + if ((str = psMetadataLookupStr(&status, config->args, "-warp_id"))) { + if (!psMetadataAddS64(config->where, PS_LIST_TAIL, "warp_id", 0, "==", (psS64)atoll(str))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item warp_id"); + psFree(config); + return NULL; + } + } +} + addWhereStr(skycell_id); + addWhereStr(tess_id); + addWhereStr(exp_tag); + + if (config->where->list->n < 1) { + psFree(config->where); + config->where = NULL; + } + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = pmConfigDB(config->modules); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +}