Index: trunk/dbconfig/cam.md =================================================================== --- trunk/dbconfig/cam.md (revision 23663) +++ trunk/dbconfig/cam.md (revision 23688) @@ -87,4 +87,5 @@ path_base STR 255 fault S16 0 # Key NOT NULL + quality S16 0 END Index: trunk/dbconfig/changes.txt =================================================================== --- trunk/dbconfig/changes.txt (revision 23663) +++ trunk/dbconfig/changes.txt (revision 23688) @@ -899,4 +899,8 @@ update chipRun SET magicked = 1 where chip_id = 11955; + +-- Version 1.1.51 + +-- Distribution updates CREATE TABLE distTarget ( target_id BIGINT AUTO_INCREMENT, @@ -908,5 +912,4 @@ PRIMARY KEY(target_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; - CREATE TABLE rcDSProduct ( @@ -931,5 +934,4 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; - CREATE TABLE rcInterest ( int_id BIGINT AUTO_INCREMENT, @@ -967,5 +969,24 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; --- 1.1.51 - ALTER TABLE distRun ADD COLUMN no_magic TINYINT AFTER clean; + + +-- Adding data quality flags +DROP TABLE guidePendingExp; +ALTER TABLE diffSkyfile ADD COLUMN quality SMALLINT NOT NULL DEFAULT 0 AFTER fault; +ALTER TABLE stackSumSkyfile ADD COLUMN quality SMALLINT NOT NULL DEFAULT 0 AFTER fault; +ALTER TABLE warpSkyfile ADD COLUMN quality SMALLINT NOT NULL DEFAULT 0 AFTER fault; +ALTER TABLE camProcessedExp ADD COLUMN quality SMALLINT NOT NULL DEFAULT 0 AFTER fault; +ALTER TABLE chipProcessedImfile ADD COLUMN quality SMALLINT NOT NULL DEFAULT 0 AFTER fault; +ALTER TABLE rawImfile ADD COLUMN quality SMALLINT NOT NULL DEFAULT 0 AFTER fault; + +ALTER TABLE warpSkyfile DROP COLUMN ignored; + +ALTER TABLE diffSkyfile ADD KEY(quality); +ALTER TABLE stackSumSkyfile ADD KEY(quality); +ALTER TABLE warpSkyfile ADD KEY(quality); +ALTER TABLE camProcessedExp ADD KEY(quality); +ALTER TABLE chipProcessedImfile ADD KEY(quality); +ALTER TABLE rawImfile ADD KEY(quality); + +-- Version 1.1.52 Index: trunk/dbconfig/chip.md =================================================================== --- trunk/dbconfig/chip.md (revision 23663) +++ trunk/dbconfig/chip.md (revision 23688) @@ -88,4 +88,5 @@ path_base STR 255 fault S16 0 # Key NOT NULL + quality S16 0 magicked BOOL f END Index: trunk/dbconfig/config.md =================================================================== --- trunk/dbconfig/config.md (revision 23663) +++ trunk/dbconfig/config.md (revision 23688) @@ -2,4 +2,4 @@ pkg_name STR ippdb pkg_namespace STR ippdb - pkg_version STR 1.1.50 + pkg_version STR 1.1.52 END Index: trunk/dbconfig/diff.md =================================================================== --- trunk/dbconfig/diff.md (revision 23663) +++ trunk/dbconfig/diff.md (revision 23688) @@ -54,4 +54,5 @@ good_frac F32 0.0 # Key fault S16 0 # Key + quality S16 0 magicked BOOL f END Index: trunk/dbconfig/guide.md =================================================================== --- trunk/dbconfig/guide.md (revision 23663) +++ (revision ) @@ -1,5 +1,0 @@ -guidePendingExp METADATA - guide_id S64 0 # Primary Key AUTO_INCREMENT - exp_id S64 64 # Key - recipe STR 64 -END Index: trunk/dbconfig/ipp.m4 =================================================================== --- trunk/dbconfig/ipp.m4 (revision 23663) +++ trunk/dbconfig/ipp.m4 (revision 23688) @@ -9,6 +9,7 @@ dnl include(telescope.md) dnl include(skycell.md) -include(tasks.md) -include(guide.md) +include(summitcopy.md) +include(new.md) +include(raw.md) include(chip.md) include(cam.md) Index: trunk/dbconfig/new.md =================================================================== --- trunk/dbconfig/new.md (revision 23688) +++ trunk/dbconfig/new.md (revision 23688) @@ -0,0 +1,25 @@ +newExp METADATA + exp_id S64 0 # Primary Key AUTO_INCREMENT + tmp_exp_name STR 64 # Key + tmp_camera STR 64 # Key + tmp_telescope STR 64 # Key + state STR 64 # Key + workdir STR 255 # destination for output files + workdir_state STR 64 # key + reduction STR 64 # Reduction class + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key + label STR 64 # Key + epoch UTC 0001-01-01T00:00:00Z +END + +# class needs to be carried here so it can go into rawImfile and be normalized +# from there +newImfile METADATA + exp_id S64 64 # Primary Key fkey(exp_id) ref newExp(exp_id) + tmp_class_id STR 64 # Primary Key + uri STR 255 + epoch UTC 0001-01-01T00:00:00Z +END + Index: trunk/dbconfig/raw.md =================================================================== --- trunk/dbconfig/raw.md (revision 23688) +++ trunk/dbconfig/raw.md (revision 23688) @@ -0,0 +1,138 @@ +# paired with rawImfile +rawExp METADATA + exp_id S64 64 # Primary Key fkey(exp_id) ref newExp(exp_id) + exp_name STR 64 # Key + camera STR 64 + telescope STR 64 + dateobs UTC 0001-01-01T00:00:00Z + exp_tag STR 255 + exp_type STR 64 + filelevel STR 64 + workdir STR 255 # destination for output files + reduction STR 64 # Reduction class + dvodb STR 255 + tess_id STR 64 + end_stage STR 64 # Key + filter STR 64 + comment STR 80 + obs_mode STR 64 # data usage goal (eg, survey name, engineering, etc) + obs_group STR 64 # identifier for data block (eg, observation sequence) + airmass F32 0.0 + ra F64 0.0 + decl F64 0.0 + exp_time F32 0.0 + sat_pixel_frac F32 0.0 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + alt F64 0.0 + az F64 0.0 + ccd_temp F32 0.0 + posang F64 0.0 + m1_x F32 0.0 + m1_y F32 0.0 + m1_z F32 0.0 + m1_tip F32 0.0 + m1_tilt F32 0.0 + m2_x F32 0.0 + m2_y F32 0.0 + m2_z F32 0.0 + m2_tip F32 0.0 + m2_tilt F32 0.0 + env_temperature F32 0.0 + env_humidity F32 0.0 + env_wind_speed F32 0.0 + env_wind_dir F32 0.0 + teltemp_m1 F32 0.0 + teltemp_m1cell F32 0.0 + teltemp_m2 F32 0.0 + teltemp_spider F32 0.0 + teltemp_truss F32 0.0 + teltemp_extra F32 0.0 + pon_time F32 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + object STR 64 + sun_angle F32 0.0 + sun_alt F32 0.0 + moon_angle F32 0.0 + moon_alt F32 0.0 + moon_phase F32 0.0 + hostname STR 64 + fault S16 0 # Key NOT NULL + epoch UTC 0001-01-01T00:00:00Z + magicked BOOL f +END + +rawImfile METADATA + exp_id S64 64 # Primary Key fkey(exp_id, tmp_class_id) ref newImfile(exp_id, tmp_class_id) + exp_name STR 64 # UINDEX(exp_id, tmp_class_id) + camera STR 64 + telescope STR 64 + dateobs UTC 0001-01-01T00:00:00Z + tmp_class_id STR 64 # Key + class_id STR 64 # Primary Key + uri STR 255 + exp_type STR 64 +# This field is used to set the per exp filelevel. Thus the values for all of +# the imfiles in an exposure need to be sanity checked to make sure that this +# value is in argeement. + filelevel STR 64 + filter STR 64 + comment STR 80 + obs_mode STR 64 # data usage goal (eg, survey name, engineering, etc) + obs_group STR 64 # identifier for data block (eg, observation sequence) + airmass F32 0.0 + ra F64 0.0 + decl F64 0.0 + exp_time F32 0.0 + sat_pixel_frac F32 0.0 + bg F64 0.0 + bg_stdev F64 0.0 + bg_mean_stdev F64 0.0 + alt F64 0.0 + az F64 0.0 + ccd_temp F32 0.0 + posang F64 0.0 + m1_x F32 0.0 + m1_y F32 0.0 + m1_z F32 0.0 + m1_tip F32 0.0 + m1_tilt F32 0.0 + m2_x F32 0.0 + m2_y F32 0.0 + m2_z F32 0.0 + m2_tip F32 0.0 + m2_tilt F32 0.0 + env_temperature F32 0.0 + env_humidity F32 0.0 + env_wind_speed F32 0.0 + env_wind_dir F32 0.0 + teltemp_m1 F32 0.0 + teltemp_m1cell F32 0.0 + teltemp_m2 F32 0.0 + teltemp_spider F32 0.0 + teltemp_truss F32 0.0 + teltemp_extra F32 0.0 + pon_time F32 0.0 + user_1 F64 0.0 + user_2 F64 0.0 + user_3 F64 0.0 + user_4 F64 0.0 + user_5 F64 0.0 + object STR 64 + sun_angle F32 0.0 + sun_alt F32 0.0 + moon_angle F32 0.0 + moon_alt F32 0.0 + moon_phase F32 0.0 + ignored BOOL f + hostname STR 64 + fault S16 0 # Key NOT NULL + quality S16 0 + epoch UTC 0001-01-01T00:00:00Z + magicked BOOL f +END Index: trunk/dbconfig/stack.md =================================================================== --- trunk/dbconfig/stack.md (revision 23663) +++ trunk/dbconfig/stack.md (revision 23688) @@ -46,3 +46,4 @@ good_frac F32 0.0 # Key fault S16 0 # Key + quality S16 0 END Index: trunk/dbconfig/summitcopy.md =================================================================== --- trunk/dbconfig/summitcopy.md (revision 23688) +++ trunk/dbconfig/summitcopy.md (revision 23688) @@ -0,0 +1,60 @@ +pzDataStore METADATA + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + uri STR 255 + epoch UTC 0001-01-01T00:00:00Z +END + +# list of source exposures -- updated as exposures are seen +# summitExp.imfiles is updated as filesets are queried default value should be +# -1 as NULL or 0 might be a valid value (empty fileset) +summitExp METADATA + exp_name STR 64 # Primary Key + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + dateobs UTC NULL + exp_type STR 64 + uri STR 255 + imfiles S32 0 + fault S16 0 # Key NOT NULL + epoch UTC 0001-01-01T00:00:00Z +END + +# class == type of file +# class_id == type set id +# list of source images -- updated as exposures/filesets are queried +summitImfile METADATA + exp_name STR 64 # Primary Key fkey(exp_name, camera, telescope) ref summitExp(exp_name, camera, telescope) + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + file_id STR 64 # Key + bytes S32 0 + md5sum STR 32 + class STR 64 # Primary Key + class_id STR 64 # Primary Key + uri STR 255 + epoch UTC 0001-01-01T00:00:00Z +END + +# list of exposures that have had their imfiles/files registered (but not +# downloaded) +pzDownloadExp METADATA + exp_name STR 64 # Primary Key fkey(exp_name, camera, telescope) ref summitExp(exp_name, camera, telescope) + camera STR 64 # Primary Key + telescope STR 64 # Primary Key + state STR 64 # Key + epoch UTC 0001-01-01T00:00:00Z +END + +pzDownloadImfile METADATA + exp_name STR 64 # Primary Key fkey(exp_name, camera, telescope) ref pzDownloadExp(exp_name, camera, telescope) + camera STR 64 # Primary Key fkey(exp_name, camera, telescope, class, class_id) ref summitImfile(exp_name, camera, telescope, class, class_id) + telescope STR 64 # Primary Key + class STR 64 # Primary Key + class_id STR 64 # Primary Key + uri STR 255 + fault S16 0 # Key NOT NULL + epoch UTC 0001-01-01T00:00:00Z + hostname STR 64 +END + Index: trunk/dbconfig/tasks.md =================================================================== --- trunk/dbconfig/tasks.md (revision 23663) +++ (revision ) @@ -1,243 +1,0 @@ -# $Id: tasks.md,v 1.159 2008-12-13 20:17:34 bills Exp $ - -# this table records all exposure ID ever seen from the summit -# exp_name == fileset - -# note that dec is a MySQL reserved word -# note that use is a MySQL reserved word -# note that exp is a MySQL reserved word - -# for use with this stored procedure to generate exp_ids under Mysql 5+ -# -#CREATE FUNCTION genTag (exp_name varchar(64)) RETURNS VARCHAR(64) -#BEGIN -# UPDATE expTagCounter SET counter = LAST_INSERT_ID(counter + 1); -# RETURN CONCAT(exp_name, '.', LAST_INSERT_ID()); -#END -# -#expTagCounter METADATA -# counter U64 0 -#END - -pzDataStore METADATA - camera STR 64 # Primary Key - telescope STR 64 # Primary Key - uri STR 255 - epoch UTC 0001-01-01T00:00:00Z -END - -# list of source exposures -- updated as exposures are seen -# summitExp.imfiles is updated as filesets are queried default value should be -# -1 as NULL or 0 might be a valid value (empty fileset) -summitExp METADATA - exp_name STR 64 # Primary Key - camera STR 64 # Primary Key - telescope STR 64 # Primary Key - dateobs UTC NULL - exp_type STR 64 - uri STR 255 - imfiles S32 0 - fault S16 0 # Key NOT NULL - epoch UTC 0001-01-01T00:00:00Z -END - -# class == type of file -# class_id == type set id -# list of source images -- updated as exposures/filesets are queried -summitImfile METADATA - exp_name STR 64 # Primary Key fkey(exp_name, camera, telescope) ref summitExp(exp_name, camera, telescope) - camera STR 64 # Primary Key - telescope STR 64 # Primary Key - file_id STR 64 # Key - bytes S32 0 - md5sum STR 32 - class STR 64 # Primary Key - class_id STR 64 # Primary Key - uri STR 255 - epoch UTC 0001-01-01T00:00:00Z -END - -# list of exposures that have had their imfiles/files registered (but not -# downloaded) -pzDownloadExp METADATA - exp_name STR 64 # Primary Key fkey(exp_name, camera, telescope) ref summitExp(exp_name, camera, telescope) - camera STR 64 # Primary Key - telescope STR 64 # Primary Key - state STR 64 # Key - epoch UTC 0001-01-01T00:00:00Z -END - -pzDownloadImfile METADATA - exp_name STR 64 # Primary Key fkey(exp_name, camera, telescope) ref pzDownloadExp(exp_name, camera, telescope) - camera STR 64 # Primary Key fkey(exp_name, camera, telescope, class, class_id) ref summitImfile(exp_name, camera, telescope, class, class_id) - telescope STR 64 # Primary Key - class STR 64 # Primary Key - class_id STR 64 # Primary Key - uri STR 255 - fault S16 0 # Key NOT NULL - epoch UTC 0001-01-01T00:00:00Z - hostname STR 64 -END - -newExp METADATA - exp_id S64 0 # Primary Key AUTO_INCREMENT - tmp_exp_name STR 64 # Key - tmp_camera STR 64 # Key - tmp_telescope STR 64 # Key - state STR 64 # Key - workdir STR 255 # destination for output files - workdir_state STR 64 # key - reduction STR 64 # Reduction class - dvodb STR 255 - tess_id STR 64 - end_stage STR 64 # Key - label STR 64 # Key - epoch UTC 0001-01-01T00:00:00Z -END - -# class needs to be carried here so it can go into rawImfile and be normalized -# from there -newImfile METADATA - exp_id S64 64 # Primary Key fkey(exp_id) ref newExp(exp_id) - tmp_class_id STR 64 # Primary Key - uri STR 255 - epoch UTC 0001-01-01T00:00:00Z -END - -# paired with rawImfile -rawExp METADATA - exp_id S64 64 # Primary Key fkey(exp_id) ref newExp(exp_id) - exp_name STR 64 # Key - camera STR 64 - telescope STR 64 - dateobs UTC 0001-01-01T00:00:00Z - exp_tag STR 255 - exp_type STR 64 - filelevel STR 64 - workdir STR 255 # destination for output files - reduction STR 64 # Reduction class - dvodb STR 255 - tess_id STR 64 - end_stage STR 64 # Key - filter STR 64 - comment STR 80 - obs_mode STR 64 # data usage goal (eg, survey name, engineering, etc) - obs_group STR 64 # identifier for data block (eg, observation sequence) - airmass F32 0.0 - ra F64 0.0 - decl F64 0.0 - exp_time F32 0.0 - sat_pixel_frac F32 0.0 - bg F64 0.0 - bg_stdev F64 0.0 - bg_mean_stdev F64 0.0 - alt F64 0.0 - az F64 0.0 - ccd_temp F32 0.0 - posang F64 0.0 - m1_x F32 0.0 - m1_y F32 0.0 - m1_z F32 0.0 - m1_tip F32 0.0 - m1_tilt F32 0.0 - m2_x F32 0.0 - m2_y F32 0.0 - m2_z F32 0.0 - m2_tip F32 0.0 - m2_tilt F32 0.0 - env_temperature F32 0.0 - env_humidity F32 0.0 - env_wind_speed F32 0.0 - env_wind_dir F32 0.0 - teltemp_m1 F32 0.0 - teltemp_m1cell F32 0.0 - teltemp_m2 F32 0.0 - teltemp_spider F32 0.0 - teltemp_truss F32 0.0 - teltemp_extra F32 0.0 - pon_time F32 0.0 - user_1 F64 0.0 - user_2 F64 0.0 - user_3 F64 0.0 - user_4 F64 0.0 - user_5 F64 0.0 - object STR 64 - sun_angle F32 0.0 - sun_alt F32 0.0 - moon_angle F32 0.0 - moon_alt F32 0.0 - moon_phase F32 0.0 - hostname STR 64 - fault S16 0 # Key NOT NULL - epoch UTC 0001-01-01T00:00:00Z - magicked BOOL f -END - -rawImfile METADATA - exp_id S64 64 # Primary Key fkey(exp_id, tmp_class_id) ref newImfile(exp_id, tmp_class_id) - exp_name STR 64 # UINDEX(exp_id, tmp_class_id) - camera STR 64 - telescope STR 64 - dateobs UTC 0001-01-01T00:00:00Z - tmp_class_id STR 64 # Key - class_id STR 64 # Primary Key - uri STR 255 - exp_type STR 64 -# This field is used to set the per exp filelevel. Thus the values for all of -# the imfiles in an exposure need to be sanity checked to make sure that this -# value is in argeement. - filelevel STR 64 - filter STR 64 - comment STR 80 - obs_mode STR 64 # data usage goal (eg, survey name, engineering, etc) - obs_group STR 64 # identifier for data block (eg, observation sequence) - airmass F32 0.0 - ra F64 0.0 - decl F64 0.0 - exp_time F32 0.0 - sat_pixel_frac F32 0.0 - bg F64 0.0 - bg_stdev F64 0.0 - bg_mean_stdev F64 0.0 - alt F64 0.0 - az F64 0.0 - ccd_temp F32 0.0 - posang F64 0.0 - m1_x F32 0.0 - m1_y F32 0.0 - m1_z F32 0.0 - m1_tip F32 0.0 - m1_tilt F32 0.0 - m2_x F32 0.0 - m2_y F32 0.0 - m2_z F32 0.0 - m2_tip F32 0.0 - m2_tilt F32 0.0 - env_temperature F32 0.0 - env_humidity F32 0.0 - env_wind_speed F32 0.0 - env_wind_dir F32 0.0 - teltemp_m1 F32 0.0 - teltemp_m1cell F32 0.0 - teltemp_m2 F32 0.0 - teltemp_spider F32 0.0 - teltemp_truss F32 0.0 - teltemp_extra F32 0.0 - pon_time F32 0.0 - user_1 F64 0.0 - user_2 F64 0.0 - user_3 F64 0.0 - user_4 F64 0.0 - user_5 F64 0.0 - object STR 64 - sun_angle F32 0.0 - sun_alt F32 0.0 - moon_angle F32 0.0 - moon_alt F32 0.0 - moon_phase F32 0.0 - ignored BOOL f - hostname STR 64 - fault S16 0 # Key NOT NULL - epoch UTC 0001-01-01T00:00:00Z - magicked BOOL f -END Index: trunk/dbconfig/warp.md =================================================================== --- trunk/dbconfig/warp.md (revision 23663) +++ trunk/dbconfig/warp.md (revision 23688) @@ -55,6 +55,6 @@ ymin S32 0 ymax S32 0 - ignored BOOL f # Key fault S16 0 # Key + quality S16 0 magicked BOOL f END