Index: /branches/eam_branches/20090820/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- /branches/eam_branches/20090820/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 25869) +++ /branches/eam_branches/20090820/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 25870) @@ -108,14 +108,8 @@ } elsif ($req_type eq "bycoord") { - # run - # regtool -dbname $image_db -processedimfile -time_begin $mjd_min -time_end = $mjd_max -filter $filter - # - my $results = lookup_bycoord($ipprc, $image_db, $x, $y, $dateobs_begin, $dateobs_end, $filter, $verbose); - - # now take the results and lookup byexp - # XXX: This needs work. What stops results from returning multiple images???? - # We should loop over the set of images returned building up the full results array - $req_type = "byexp"; - $id = $results->{exp_name}; + if ($img_type eq "stack") { + print STDERR "Error bycoord lookup not implemented for stage stack yet\n"; + return undef; + } } elsif ($req_type eq "byskycell") { if (($img_type eq "raw") or ($img_type eq "chip")) { @@ -129,10 +123,11 @@ } - my $results = lookup($ipprc, $image_db, $req_type, $img_type, $id, $tess_id, $component, $need_magic, $dateobs_begin, $dateobs_end, $filter, $label, $verbose); + my $results = lookup($ipprc, $image_db, $req_type, $img_type, $id, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $label, $verbose); return $results; } -sub lookup { +sub lookup +{ my $ipprc = shift; my $image_db = shift; @@ -143,4 +138,6 @@ my $component= shift; my $need_magic = shift; + my $x = shift; + my $y = shift; my $dateobs_begin = shift; my $dateobs_end = shift; @@ -175,9 +172,13 @@ my $magic_arg = $need_magic ? " -destreaked" : ""; + my $component_args; if ($img_type eq "raw") { $class_id = $component; $command = "$regtool -processedimfile -dbname $image_db"; + # XXX: for now restrict lookups to type object + # are stamps of detrend exposures interesting? + $command .= " -exp_type object"; $id_opt = "-exp_id"; - $command .= " -class_id $class_id" if $component; + $component_args = " -class_id $class_id" if $component; $want_astrom = 1; $set_class_id = 1; @@ -185,5 +186,5 @@ $class_id = $component; $command = "$chiptool -processedimfile -dbname $image_db"; - $command .= " -class_id $class_id" if $class_id; + $component_args = " -class_id $class_id" if $class_id; $id_opt = "-chip_id"; $image_name = "PPIMAGE.CHIP"; @@ -196,5 +197,5 @@ $skycell_id = $component; $command = "$warptool -warped -dbname $image_db"; - $command .= " -skycell_id $skycell_id" if $skycell_id; + $component_args = " -skycell_id $skycell_id" if $skycell_id; $id_opt = "-warp_id"; $image_name = "PSWARP.OUTPUT"; @@ -205,5 +206,5 @@ $skycell_id = $component; $command = "$difftool -diffskyfile -dbname $image_db"; - $command .= " -skycell_id $skycell_id" if $skycell_id; + $component_args = " -skycell_id $skycell_id" if $skycell_id; $id_opt = "-diff_id"; $image_name = "PPSUB.OUTPUT"; @@ -212,6 +213,8 @@ $base_name = "path_base"; } elsif ($img_type eq "stack") { + $skycell_id = $component; $command = "$stacktool -sumskyfile -dbname $image_db"; $id_opt = "-stack_id"; + $component_args = " -skycell_id $skycell_id" if $skycell_id; $image_name = "PPSTACK.OUTPUT"; @@ -225,8 +228,13 @@ if ($req_type eq "byid") { $command .= " $id_opt $id"; + $command .= $component_args if $component_args; } elsif ($req_type eq "byexp") { $command .= " -exp_name $id"; + $command .= $component_args if $component_args; } elsif ($req_type eq "byskycell") { + die "tess_id and component are required for byskycell" if !$tess_id or ! $skycell_id; $command .= " -tess_id $tess_id -skycell_id $skycell_id"; + } elsif ($req_type eq "bycoord") { + $command .= " -radius 3.0 -ra $x -decl $y"; } else { die "Unknown req_type supplied: $req_type"; @@ -289,5 +297,5 @@ # if uri is nil this will get overridded below - # (this is for raw stage) + # (we do this here for raw stage) $out->{image} = $image->{uri}; if ($set_class_id) { @@ -295,8 +303,23 @@ $out->{component} = $class_id; } + my $stage_id; + if ($img_type eq "raw") { + $stage_id = $image->{exp_id}; + } elsif ($img_type eq "chip") { + $stage_id = $image->{chip}; + } elsif ($img_type eq "warp") { + $stage_id = $image->{warp_id}; + } elsif ($img_type eq "diff") { + $stage_id = $image->{diff_id}; + } elsif ($img_type eq "stack") { + $stage_id = $image->{stack_id}; + } + $image->{stage_id} = $stage_id; + $image->{stage} = $img_type; + $image->{image_db} = $image_db; # find the mask and weight images if ($base) { - $out->{image} = $ipprc->filename($image_name, $base, $class_id) if $image_name; + $out->{image} = $ipprc->filename($image_name, $base, $class_id) if $image_name; $out->{mask} = $ipprc->filename($mask_name, $base, $class_id) if $mask_name; $out->{weight} = $ipprc->filename($weight_name, $base, $class_id) if $weight_name; Index: /branches/eam_branches/20090820/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm =================================================================== --- /branches/eam_branches/20090820/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 25869) +++ /branches/eam_branches/20090820/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 25870) @@ -23,4 +23,5 @@ $PSTAMP_SELECT_WEIGHT $PSTAMP_SELECT_INVERSE + $PSTAMP_WAIT_FOR_UPDATE $PSTAMP_SUCCESS $PSTAMP_SYSTEM_ERROR @@ -49,4 +50,5 @@ our $PSTAMP_SELECT_INVERSE = 1024; +our $PSTAMP_WAIT_FOR_UPDATE = 2048; # job and result codes Index: /branches/eam_branches/20090820/dbconfig/add.md =================================================================== --- /branches/eam_branches/20090820/dbconfig/add.md (revision 25869) +++ /branches/eam_branches/20090820/dbconfig/add.md (revision 25870) @@ -1,11 +1,15 @@ +# XXX rename these 'addstarRun', etc... (add_id -> addstar_id) addRun METADATA add_id S64 0 # Primary Key AUTO_INCREMENT cam_id S64 0 # Key INDEX(add_id,cam_id) fkey(cam_id) ref camRun(cam_id) - state STR 64 # key + state STR 64 # Key workdir STR 255 workdir_state STR 64 + reduction STR 64 label STR 64 + data_group STR 64 # Key dvodb STR 255 - magicked S64 0 + note STR 255 + image_only BOOL f END @@ -13,7 +17,4 @@ add_id S64 0 # Primary Key AUTO_INCREMENT dtime_addstar F32 0.0 - - n_stars S32 0 - path_base STR 255 fault S16 0 # Key NOT NULL Index: /branches/eam_branches/20090820/dbconfig/cam.md =================================================================== --- /branches/eam_branches/20090820/dbconfig/cam.md (revision 25869) +++ /branches/eam_branches/20090820/dbconfig/cam.md (revision 25870) @@ -6,4 +6,6 @@ workdir_state STR 64 # key label STR 64 # key + data_group STR 64 # key + dist_group STR 64 # key reduction STR 64 expgroup STR 64 # key @@ -12,4 +14,5 @@ end_stage STR 64 # Key magicked S64 0 + note STR 255 END Index: /branches/eam_branches/20090820/dbconfig/changes.1.1.56 =================================================================== --- /branches/eam_branches/20090820/dbconfig/changes.1.1.56 (revision 25870) +++ /branches/eam_branches/20090820/dbconfig/changes.1.1.56 (revision 25870) @@ -0,0 +1,78 @@ +ALTER TABLE chipRun ADD COLUMN data_group VARCHAR(64) AFTER label; +ALTER TABLE chipRun ADD COLUMN dist_group VARCHAR(64) AFTER data_group; +ALTER TABLE chipRun ADD COLUMN note VARCHAR(255) AFTER magicked; +ALTER TABLE chipRun ADD KEY(data_group); +ALTER TABLE chipRun ADD KEY(dist_group); + +UPDATE chipRun SET data_group = label; +UPDATE chipRun SET dist_group = label; + +ALTER TABLE camRun ADD COLUMN data_group VARCHAR(64) AFTER label; +ALTER TABLE camRun ADD COLUMN dist_group VARCHAR(64) AFTER data_group; +ALTER TABLE camRun ADD COLUMN note VARCHAR(255) AFTER magicked; +ALTER TABLE camRun ADD KEY(data_group); +ALTER TABLE camRun ADD KEY(dist_group); + +UPDATE camRun SET data_group = label; +UPDATE camRun SET dist_group = label; + +ALTER TABLE addRun ADD COLUMN data_group VARCHAR(64) AFTER label; +ALTER TABLE addRun ADD COLUMN note VARCHAR(255) AFTER dvodb; +ALTER TABLE addRun ADD KEY(data_group); + +UPDATE addRun SET data_group = label; + +ALTER TABLE fakeRun ADD COLUMN data_group VARCHAR(64) AFTER label; +ALTER TABLE fakeRun ADD COLUMN dist_group VARCHAR(64) AFTER data_group; +ALTER TABLE fakeRun ADD COLUMN note VARCHAR(255) AFTER epoch; +ALTER TABLE fakeRun ADD KEY(data_group); +ALTER TABLE fakeRun ADD KEY(dist_group); + +UPDATE fakeRun SET data_group = label; +UPDATE fakeRun SET dist_group = label; + +ALTER TABLE warpRun ADD COLUMN data_group VARCHAR(64) AFTER label; +ALTER TABLE warpRun ADD COLUMN dist_group VARCHAR(64) AFTER data_group; +ALTER TABLE warpRun ADD COLUMN note VARCHAR(255) AFTER magicked; +ALTER TABLE warpRun ADD KEY(data_group); +ALTER TABLE warpRun ADD KEY(dist_group); + +UPDATE warpRun SET data_group = label; +UPDATE warpRun SET dist_group = label; + +ALTER TABLE stackRun ADD COLUMN data_group VARCHAR(64) AFTER label; +ALTER TABLE stackRun ADD COLUMN dist_group VARCHAR(64) AFTER data_group; +ALTER TABLE stackRun ADD COLUMN note VARCHAR(255) AFTER filter; +ALTER TABLE stackRun ADD KEY(data_group); +ALTER TABLE stackRun ADD KEY(dist_group); + +UPDATE stackRun SET data_group = label; +UPDATE stackRun SET dist_group = label; + +ALTER TABLE diffRun ADD COLUMN data_group VARCHAR(64) AFTER label; +ALTER TABLE diffRun ADD COLUMN dist_group VARCHAR(64) AFTER data_group; +ALTER TABLE diffRun ADD COLUMN note VARCHAR(255) AFTER magicked; +ALTER TABLE diffRun ADD KEY(data_group); +ALTER TABLE diffRun ADD KEY(dist_group); + +UPDATE diffRun SET data_group = label; +UPDATE diffRun SET dist_group = label; + +ALTER TABLE magicRun ADD COLUMN data_group VARCHAR(64) AFTER label; +ALTER TABLE magicRun ADD COLUMN note VARCHAR(255) AFTER fault; +ALTER TABLE magicRun ADD KEY(data_group); + +UPDATE magicRun SET data_group = label; + +ALTER TABLE magicDSRun ADD COLUMN data_group VARCHAR(64) AFTER label; +ALTER TABLE magicDSRun ADD COLUMN fault SMALLINT after remove; +ALTER TABLE magicDSRun ADD COLUMN note VARCHAR(255) AFTER fault; +ALTER TABLE magicDSRun ADD KEY(fault); +ALTER TABLE magicDSRun ADD KEY(data_group); + +UPDATE magicDSRun SET data_group = label; +UPDATE magicDSRun SET fault = 0; + +ALTER TABLE distRun ADD COLUMN note VARCHAR(255) after fault; + +ALTER TABLE distTarget CHANGE COLUMN label dist_group VARCHAR(64); Index: /branches/eam_branches/20090820/dbconfig/changes.txt =================================================================== --- /branches/eam_branches/20090820/dbconfig/changes.txt (revision 25869) +++ /branches/eam_branches/20090820/dbconfig/changes.txt (revision 25870) @@ -1300,2 +1300,126 @@ ALTER TABLE pstampJob ADD COLUMN options BIGINT; + +ALTER TABLE addRun ADD COLUMN reduction VARCHAR(64) AFTER workdir_state; +ALTER TABLE addRun DROP COLUMN magicked; +ALTER TABLE addRun ADD COLUMN image_only tinyint after dvodb; +ALTER TABLE addProcessedExp DROP COLUMN n_stars; + +CREATE TABLE flatcorrAddstarLink ( + corr_id BIGINT, + cam_id BIGINT, + add_id BIGINT, + include TINYINT, + PRIMARY KEY(corr_id, cam_id, add_id), + FOREIGN KEY (corr_id) REFERENCES flatcorrRun(corr_id), + FOREIGN KEY (cam_id) REFERENCES camRun(cam_id), + FOREIGN KEY (add_id) REFERENCES addRun(add_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +ALTER TABLE pstampJob ADD COLUMN dep_id BIGINT; +CREATE TABLE pstampDependent ( + dep_id BIGINT AUTO_INCREMENT, + state VARCHAR(64), + stage VARCHAR(64), + stage_id BIGINT, + imagedb VARCHAR(64), + rlabel VARCHAR(64), + no_magic TINYINT, + PRIMARY KEY(dep_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + +ALTER TABLE chipRun ADD COLUMN data_group VARCHAR(64) AFTER label; +ALTER TABLE chipRun ADD COLUMN dist_group VARCHAR(64) AFTER data_group; +ALTER TABLE chipRun ADD COLUMN note VARCHAR(255) AFTER magicked; +ALTER TABLE chipRun ADD KEY(data_group); +ALTER TABLE chipRun ADD KEY(dist_group); + +UPDATE chipRun SET data_group = label; +UPDATE chipRun SET dist_group = label; + +ALTER TABLE camRun ADD COLUMN data_group VARCHAR(64) AFTER label; +ALTER TABLE camRun ADD COLUMN dist_group VARCHAR(64) AFTER data_group; +ALTER TABLE camRun ADD COLUMN note VARCHAR(255) AFTER magicked; +ALTER TABLE camRun ADD KEY(data_group); +ALTER TABLE camRun ADD KEY(dist_group); + +UPDATE camRun SET data_group = label; +UPDATE camRun SET dist_group = label; + +ALTER TABLE addRun ADD COLUMN data_group VARCHAR(64) AFTER label; +ALTER TABLE addRun ADD COLUMN note VARCHAR(255) AFTER dvodb; +ALTER TABLE addRun ADD KEY(data_group); + +UPDATE addRun SET data_group = label; + +ALTER TABLE fakeRun ADD COLUMN data_group VARCHAR(64) AFTER label; +ALTER TABLE fakeRun ADD COLUMN dist_group VARCHAR(64) AFTER data_group; +ALTER TABLE fakeRun ADD COLUMN note VARCHAR(255) AFTER epoch; +ALTER TABLE fakeRun ADD KEY(data_group); +ALTER TABLE fakeRun ADD KEY(dist_group); + +UPDATE fakeRun SET data_group = label; +UPDATE fakeRun SET dist_group = label; + +ALTER TABLE warpRun ADD COLUMN data_group VARCHAR(64) AFTER label; +ALTER TABLE warpRun ADD COLUMN dist_group VARCHAR(64) AFTER data_group; +ALTER TABLE warpRun ADD COLUMN note VARCHAR(255) AFTER magicked; +ALTER TABLE warpRun ADD KEY(data_group); +ALTER TABLE warpRun ADD KEY(dist_group); + +UPDATE warpRun SET data_group = label; +UPDATE warpRun SET dist_group = label; + +ALTER TABLE stackRun ADD COLUMN data_group VARCHAR(64) AFTER label; +ALTER TABLE stackRun ADD COLUMN dist_group VARCHAR(64) AFTER data_group; +ALTER TABLE stackRun ADD COLUMN note VARCHAR(255) AFTER filter; +ALTER TABLE stackRun ADD KEY(data_group); +ALTER TABLE stackRun ADD KEY(dist_group); + +UPDATE stackRun SET data_group = label; +UPDATE stackRun SET dist_group = label; + +ALTER TABLE diffRun ADD COLUMN data_group VARCHAR(64) AFTER label; +ALTER TABLE diffRun ADD COLUMN dist_group VARCHAR(64) AFTER data_group; +ALTER TABLE diffRun ADD COLUMN note VARCHAR(255) AFTER magicked; +ALTER TABLE diffRun ADD KEY(data_group); +ALTER TABLE diffRun ADD KEY(dist_group); + +UPDATE diffRun SET data_group = label; +UPDATE diffRun SET dist_group = label; + +ALTER TABLE magicRun ADD COLUMN data_group VARCHAR(64) AFTER label; +ALTER TABLE magicRun ADD COLUMN note VARCHAR(255) AFTER fault; +ALTER TABLE magicRun ADD KEY(data_group); + +UPDATE magicRun SET data_group = label; + +ALTER TABLE magicDSRun ADD COLUMN data_group VARCHAR(64) AFTER label; +ALTER TABLE magicDSRun ADD COLUMN fault SMALLINT after remove; +ALTER TABLE magicDSRun ADD COLUMN note VARCHAR(255) AFTER fault; +ALTER TABLE magicDSRun ADD KEY(fault); +ALTER TABLE magicDSRun ADD KEY(data_group); + +UPDATE magicDSRun SET data_group = label; +UPDATE magicDSRun SET fault = 0; + +ALTER TABLE distRun ADD COLUMN note VARCHAR(255) after fault; + +ALTER TABLE distTarget CHANGE COLUMN label dist_group VARCHAR(64); + +ALTER TABLE rcDestination ADD CONSTRAINT UNIQUE(name); + +CREATE TABLE dbversion ( + schema_version VARCHAR(64), + updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP +) ENGINE=innodb DEFAULT CHARSET=latin1; + +INSERT INTO dbversion VALUES('1.1.56', CURRENT_TIMESTAMP); + + +-- Vesion 1.1.57 + +-- add changes for 1.1.57 here. Then uncomment the following before commiting + +-- UPDATE dbversion set schema_version = '1.1.57', updated= CURRENT_TIMESTAMP(); Index: /branches/eam_branches/20090820/dbconfig/chip.md =================================================================== --- /branches/eam_branches/20090820/dbconfig/chip.md (revision 25869) +++ /branches/eam_branches/20090820/dbconfig/chip.md (revision 25870) @@ -6,4 +6,6 @@ workdir_state STR 64 # Key label STR 64 # Key + data_group STR 64 # Key + dist_group STR 64 # Key reduction STR 64 # Reduction class expgroup STR 64 # Key @@ -12,4 +14,5 @@ end_stage STR 64 # Key magicked S64 0 + note STR 255 END Index: /branches/eam_branches/20090820/dbconfig/config.md =================================================================== --- /branches/eam_branches/20090820/dbconfig/config.md (revision 25869) +++ /branches/eam_branches/20090820/dbconfig/config.md (revision 25870) @@ -2,4 +2,4 @@ pkg_name STR ippdb pkg_namespace STR ippdb - pkg_version STR 1.1.55 + pkg_version STR 1.1.56 END Index: /branches/eam_branches/20090820/dbconfig/dbversion.md =================================================================== --- /branches/eam_branches/20090820/dbconfig/dbversion.md (revision 25870) +++ /branches/eam_branches/20090820/dbconfig/dbversion.md (revision 25870) @@ -0,0 +1,5 @@ +dbversion METADATA + schema_version S64 0 + updated UTC 0001-01-01T00:00:00Z +END + Index: /branches/eam_branches/20090820/dbconfig/diff.md =================================================================== --- /branches/eam_branches/20090820/dbconfig/diff.md (revision 25869) +++ /branches/eam_branches/20090820/dbconfig/diff.md (revision 25870) @@ -6,4 +6,6 @@ workdir STR 255 label STR 64 # Key + data_group STR 64 + dist_group STR 64 reduction STR 64 # Reduction class dvodb STR 255 @@ -13,4 +15,5 @@ exposure BOOL f magicked S64 0 + note STR 255 END Index: /branches/eam_branches/20090820/dbconfig/dist.md =================================================================== --- /branches/eam_branches/20090820/dbconfig/dist.md (revision 25869) +++ /branches/eam_branches/20090820/dbconfig/dist.md (revision 25870) @@ -1,5 +1,5 @@ distTarget METADATA target_id S64 0 # Primary Key - label STR 64 + dist_group STR 64 filter STR 64 stage STR 64 @@ -22,4 +22,5 @@ time_stamp UTC 0001-01-01T00:00:00Z fault S16 0 + note STR 255 END Index: /branches/eam_branches/20090820/dbconfig/fake.md =================================================================== --- /branches/eam_branches/20090820/dbconfig/fake.md (revision 25869) +++ /branches/eam_branches/20090820/dbconfig/fake.md (revision 25870) @@ -5,4 +5,6 @@ workdir STR 255 label STR 64 # Key + data_group STR 64 # Key + dist_group STR 64 # Key reduction STR 64 # Reduction class expgroup STR 64 # Key @@ -11,4 +13,5 @@ end_stage STR 64 # Key epoch UTC 0001-01-01T00:00:00Z + note STR 255 END Index: /branches/eam_branches/20090820/dbconfig/flatcorr.md =================================================================== --- /branches/eam_branches/20090820/dbconfig/flatcorr.md (revision 25869) +++ /branches/eam_branches/20090820/dbconfig/flatcorr.md (revision 25870) @@ -33,2 +33,10 @@ END +# table of exposures sent to addstar +flatcorrAddstarLink METADATA + corr_id S64 0 # Primary Key fkey(corr_id) ref flatcorrRun(corr_id) + cam_id S64 0 # Primary Key fkey(chip_id) ref chipRun(chip_id) + add_id S64 0 # Primary Key fkey(chip_id) ref chipRun(chip_id) + include BOOL t +END + Index: /branches/eam_branches/20090820/dbconfig/ipp.m4 =================================================================== --- /branches/eam_branches/20090820/dbconfig/ipp.m4 (revision 25869) +++ /branches/eam_branches/20090820/dbconfig/ipp.m4 (revision 25870) @@ -9,4 +9,5 @@ dnl include(telescope.md) dnl include(skycell.md) +include(dbversion.md) include(summitcopy.md) include(new.md) Index: /branches/eam_branches/20090820/dbconfig/magic.md =================================================================== --- /branches/eam_branches/20090820/dbconfig/magic.md (revision 25869) +++ /branches/eam_branches/20090820/dbconfig/magic.md (revision 25870) @@ -11,7 +11,9 @@ workdir_state STR 255 # Key label STR 64 # key + data_group STR 64 dvodb STR 255 registered TAI NULL fault S16 0 # Key + note STR 255 END @@ -53,8 +55,11 @@ cam_id S64 0 label STR 64 # key + data_group STR 64 # key outroot STR 255 recoveryroot STR 255 re_place BOOL f remove BOOL f + fault S16 0 # Key + note STR 255 END Index: /branches/eam_branches/20090820/dbconfig/pstamp.md =================================================================== --- /branches/eam_branches/20090820/dbconfig/pstamp.md (revision 25869) +++ /branches/eam_branches/20090820/dbconfig/pstamp.md (revision 25870) @@ -40,3 +40,14 @@ outputBase STR 255 options S64 64 + dep_id S64 0 END + +pstampDependent METADATA + dep_id S64 0 # Primary Key AUTO_INCREMENT + state STR 64 + stage STR 64 + stage_id S64 0 + imagedb STR 64 + rlabel STR 64 + no_magic BOOL f +END Index: /branches/eam_branches/20090820/dbconfig/stack.md =================================================================== --- /branches/eam_branches/20090820/dbconfig/stack.md (revision 25869) +++ /branches/eam_branches/20090820/dbconfig/stack.md (revision 25870) @@ -6,4 +6,6 @@ workdir STR 255 label STR 64 # Key + data_group STR 64 # Key + dist_group STR 64 # Key reduction STR 64 # Reduction class dvodb STR 255 @@ -12,4 +14,5 @@ tess_id STR 64 # Key filter STR 64 + note STR 255 END Index: /branches/eam_branches/20090820/dbconfig/warp.md =================================================================== --- /branches/eam_branches/20090820/dbconfig/warp.md (revision 25869) +++ /branches/eam_branches/20090820/dbconfig/warp.md (revision 25870) @@ -17,4 +17,6 @@ workdir_state STR 64 # Key label STR 64 # key + data_group STR 64 + dist_group STR 64 dvodb STR 255 tess_id STR 64 @@ -23,4 +25,5 @@ registered TAI NULL magicked S64 0 + note STR 255 END Index: /branches/eam_branches/20090820/ippMonitor/Makefile.in =================================================================== --- /branches/eam_branches/20090820/ippMonitor/Makefile.in (revision 25869) +++ /branches/eam_branches/20090820/ippMonitor/Makefile.in (revision 25870) @@ -151,6 +151,6 @@ $(DESTWWW)/calRun.php \ $(DESTWWW)/flatcorrRun.php \ -$(DESTWWW)/flatcorrExp.php - +$(DESTWWW)/flatcorrChip.php \ +$(DESTWWW)/flatcorrCamera.php PICTURES = \ Index: /branches/eam_branches/20090820/ippMonitor/def/flatcorrCamera.d =================================================================== --- /branches/eam_branches/20090820/ippMonitor/def/flatcorrCamera.d (revision 25870) +++ /branches/eam_branches/20090820/ippMonitor/def/flatcorrCamera.d (revision 25870) @@ -0,0 +1,33 @@ +TABLE flatcorrCamLink, flatcorrRun, camRun, chipRun, rawExp +TITLE flatcorr Camera Runs +FILE flatcorrCamera.php +MENU ipp.detrend.dat + +WHERE flatcorrRun.corr_id = flatcorrCamLink.corr_id +WHERE flatcorrCamLink.cam_id = camRun.cam_id +WHERE flatcorrCamLink.chip_id = chipRun.chip_id +WHERE chipRun.exp_id = rawExp.exp_id + +OP OP1 $rawExp.ra * 57.295783 +OP OP2 $rawExp.decl * 57.295783 + +# field size format name show link to extras +FIELD flatcorrRun.corr_id, 5, %s, Corr ID +FIELD rawExp.exp_name, 5, %s, Exp Name +FIELD rawExp.exp_id, 5, %d, Exp ID, value, rawImfile.php, ARG1 +FIELD chipRun.chip_id, 7, %d, Chip ID, value, chipProcessedImfile.php, ARG2 +FIELD camRun.state, 7, %s, State +FIELD chipRun.label, 7, %s, Label +FIELD rawExp.telescope, 10, %s, Telescope +FIELD rawExp.camera, 10, %s, Camera +FIELD rawExp.dateobs, 19, %T, Date/Time +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 +FIELD rawExp.object, 8, %s, Object +FIELD flatcorrRun.filter, 10, %s, FILTER +FIELD rawExp.filter, 10, %s, FILTER +FIELD rawExp.exp_time, 5, %.2f, exp_time +FIELD rawExp.airmass, 5, %.4f, airmass +FIELD rawExp.bg, 5, %.2f, backgnd +FIELD rawExp.bg_stdev, 5, %.2f, stdev +FIELD rawExp.comment, 75, %s, Comment Index: /branches/eam_branches/20090820/ippMonitor/def/flatcorrChip.d =================================================================== --- /branches/eam_branches/20090820/ippMonitor/def/flatcorrChip.d (revision 25870) +++ /branches/eam_branches/20090820/ippMonitor/def/flatcorrChip.d (revision 25870) @@ -0,0 +1,32 @@ +TABLE flatcorrChipLink, flatcorrRun, chipRun, rawExp +TITLE flatcorr Chip Runs +FILE flatcorrChip.php +MENU ipp.detrend.dat + +WHERE flatcorrRun.corr_id = flatcorrChipLink.corr_id +WHERE flatcorrChipLink.chip_id = chipRun.chip_id +WHERE chipRun.exp_id = rawExp.exp_id + +OP OP1 $rawExp.ra * 57.295783 +OP OP2 $rawExp.decl * 57.295783 + +# field size format name show link to extras +FIELD flatcorrRun.corr_id, 5, %s, Corr ID +FIELD rawExp.exp_name, 5, %s, Exp Name +FIELD rawExp.exp_id, 5, %d, Exp ID, value, rawImfile.php, ARG1 +FIELD chipRun.chip_id, 7, %d, Chip ID, value, chipProcessedImfile.php, ARG2 +FIELD chipRun.state, 7, %s, State +FIELD chipRun.label, 7, %s, Label +FIELD rawExp.telescope, 10, %s, Telescope +FIELD rawExp.camera, 10, %s, Camera +FIELD rawExp.dateobs, 19, %T, Date/Time +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 +FIELD rawExp.object, 8, %s, Object +FIELD flatcorrRun.filter, 10, %s, FILTER +FIELD rawExp.filter, 10, %s, FILTER +FIELD rawExp.exp_time, 5, %.2f, exp_time +FIELD rawExp.airmass, 5, %.4f, airmass +FIELD rawExp.bg, 5, %.2f, backgnd +FIELD rawExp.bg_stdev, 5, %.2f, stdev +FIELD rawExp.comment, 75, %s, Comment Index: /branches/eam_branches/20090820/ippMonitor/raw/ipp.detrend.dat =================================================================== --- /branches/eam_branches/20090820/ippMonitor/raw/ipp.detrend.dat (revision 25869) +++ /branches/eam_branches/20090820/ippMonitor/raw/ipp.detrend.dat (revision 25870) @@ -53,3 +53,4 @@ menutop | menutop | plain | Flat Correction | menulink | menuselect | link | Flat Corr Runs | flatcorrRun.php -menulink | menuselect | link | Flat Corr Exp | flatcorrExp.php +menulink | menuselect | link | Flat Corr Chip Runs | flatcorrChip.php +menulink | menuselect | link | Flat Corr Camera Runs | flatcorrCamera.php Index: /branches/eam_branches/20090820/ippMonitor/raw/ipp.php =================================================================== --- /branches/eam_branches/20090820/ippMonitor/raw/ipp.php (revision 25869) +++ /branches/eam_branches/20090820/ippMonitor/raw/ipp.php (revision 25870) @@ -285,4 +285,7 @@ $file = fopen ($source, "r"); + global $DBHOST; + $HOST = $_SERVER['SERVER_NAME']; + echo "

host: $HOST, db: $DBHOST

\n"; echo "
\n"; echo "\n"; Index: /branches/eam_branches/20090820/ippScripts/Build.PL =================================================================== --- /branches/eam_branches/20090820/ippScripts/Build.PL (revision 25869) +++ /branches/eam_branches/20090820/ippScripts/Build.PL (revision 25870) @@ -59,4 +59,5 @@ scripts/magic_destreak.pl scripts/magic_destreak_revert.pl + scripts/magic_destreak_cleanup.pl scripts/ippdb.pl scripts/ipp_cleanup.pl Index: /branches/eam_branches/20090820/ippScripts/scripts/addstar_run.pl =================================================================== --- /branches/eam_branches/20090820/ippScripts/scripts/addstar_run.pl (revision 25869) +++ /branches/eam_branches/20090820/ippScripts/scripts/addstar_run.pl (revision 25870) @@ -28,10 +28,6 @@ # Look for programs we need my $missing_tools; -#my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); -#my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); -#my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); -#my $psastro = can_run('psastro') or (warn "Can't find psastro" and $missing_tools = 1); my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); if ($missing_tools) { @@ -40,10 +36,9 @@ } -my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $recipe, $dbname, $reduction, $dvodb, $verbose, $no_update, - $no_op, $redirect, $save_temps, $run_state); +my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $dbname, $reduction, $dvodb, $image_only, $verbose, $no_update, + $no_op, $redirect, $save_temps); GetOptions( 'exp_tag=s' => \$exp_tag, # Exposure identifier 'add_id=s' => \$add_id, # Camtool identifier - 'recipe=s' => \$recipe, # Recipe to use 'camera|c=s' => \$camera, # Camera 'dbname|d=s' => \$dbname, # Database name @@ -52,5 +47,5 @@ 'reduction=s' => \$reduction, # Reduction class 'dvodb|w=s' => \$dvodb, # output DVO database - 'run-state=s' => \$run_state, # 'new' or 'update' + 'image-only' => \$image_only, # Print to stdout 'verbose' => \$verbose, # Print to stdout 'no-update' => \$no_update, # Update the database? @@ -74,9 +69,4 @@ my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); -if (not defined $run_state) { $run_state = 'new'; } -if ($run_state eq 'update') { - $logDest .= '.update'; -} - if ($redirect) { $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $add_id, $PS_EXIT_SYS_ERROR ); @@ -89,34 +79,9 @@ $reduction = 'DEFAULT' unless defined $reduction; +# XXX This is now not used: do we still need it? my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use &my_die("Unrecognised ADDSTAR recipe", $add_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar; -#my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use -#&my_die("Unrecognised PSASTRO recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro; - my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - -my $cmdflags; - -# Get list of component files -my $files; # Array of component files -{ - my $command = "$addtool -pendingexp -add_id $add_id"; # Command to run - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform addtool: $error_code", $add_id, $error_code); - } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $add_id, $PS_EXIT_PROG_ERROR); - - # extract the metadata for the files into a hash list - $files = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); -} - -my $chipObjectsExist = 0; # Output products @@ -126,8 +91,4 @@ my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $camroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); my $traceDest = $ipprc->filename("TRACE.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); - -if ($run_state eq 'update') { - $traceDest .= '.update'; -} # convert supplied DVO database name to UNIX filename @@ -141,38 +102,28 @@ unless ($no_op) { - if (defined $dvodbReal and ($run_state eq 'new')) { - ## XXX the camera analysis can either save the full set of - ## detections, or just the image metadata, in the dvodb + if (defined $dvodbReal) { + + ## addstar can either save the full set of detections, or just + ## the image metadata, in the dvodb. this is set in the + ## database table addRun - ## get the addstar recipe for this camera and CAMERA reduction - my $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppConfigDump: $error_code", $add_id, $PS_EXIT_SYS_ERROR); - } - my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $add_id, $PS_EXIT_SYS_ERROR); - - ## allow the dvodb to save only images, or the full detection set - my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY'); - - # XXX this construct requires the user to have a valid .ptolemyrc - # XXX which in turn points at ippconfig/dvo.site + # addstar requires the user to have a valid .ptolemyrc which + # in turn points at ippconfig/dvo.site + + # get the names for the camera and the real input file + my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar + my $realFile = $ipprc->file_resolve($fpaObjects); + # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) - # XXX this needs to be converted to addstar_client... - - my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar - $command = "$addstar -D CAMERA $camdir -update"; - $command .= " -image" if $imagesOnly; + my $command = "$addstar -update"; # XXX optionally set -update? + $command .= " -D CAMERA $camdir"; $command .= " -D CATDIR $dvodbReal"; - - my $realFile = $ipprc->file_resolve($fpaObjects); $command .= " $realFile"; + $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference + $command .= " -image" if $image_only; my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { @@ -184,15 +135,9 @@ } - # This needs to be updated when addtool is written. BROKEN my $fpaCommand = "$addtool -add_id $add_id"; -if ($run_state eq 'new') { - $fpaCommand .= " -addprocessedexp"; - $fpaCommand .= " -path_base $outroot"; - $fpaCommand .= " $cmdflags"; - $fpaCommand .= " -dtime_addstar $dtime_addstar"; -} else { - $fpaCommand .= " -updaterun -state full"; -} +$fpaCommand .= " -addprocessedexp"; +$fpaCommand .= " -path_base $outroot"; +$fpaCommand .= " -dtime_addstar $dtime_addstar"; $fpaCommand .= " -dbname $dbname" if defined $dbname; @@ -223,15 +168,8 @@ # This needs to be updated when addtool is written. BROKEN my $command = "$addtool -add_id $add_id"; - if ($run_state eq 'new') { - $command .= " -addprocessedexp"; - $command .= " -uri UNKNOWN"; - $command .= " -fault $exit_code"; - $command .= " -path_base $outroot"; - $command .= " -path_base $outroot" if defined $outroot; - $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); - } else { - $command .= " -updateprocessedexp"; - $command .= " -fault $exit_code"; - } + $command .= " -addprocessedexp"; + $command .= " -fault $exit_code"; + $command .= " -path_base $outroot" if defined $outroot; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); $command .= " -dbname $dbname" if defined $dbname; system ($command); @@ -239,5 +177,4 @@ exit $exit_code; } - END { Index: /branches/eam_branches/20090820/ippScripts/scripts/camera_exp.pl =================================================================== --- /branches/eam_branches/20090820/ippScripts/scripts/camera_exp.pl (revision 25869) +++ /branches/eam_branches/20090820/ippScripts/scripts/camera_exp.pl (revision 25870) @@ -39,10 +39,9 @@ } -my ( $exp_tag, $cam_id, $camera, $outroot, $recipe, $dbname, $reduction, $dvodb, $verbose, $no_update, +my ( $exp_tag, $cam_id, $camera, $outroot, $dbname, $reduction, $dvodb, $verbose, $no_update, $no_op, $redirect, $save_temps, $run_state); GetOptions( 'exp_tag=s' => \$exp_tag, # Exposure identifier 'cam_id=s' => \$cam_id, # Camtool identifier - 'recipe=s' => \$recipe, # Recipe to use 'camera|c=s' => \$camera, # Camera 'dbname|d=s' => \$dbname, # Database name @@ -355,5 +354,5 @@ # $fpaCommand .= " -dtime_addstar $dtime_addstar"; } else { - $fpaCommand .= " -updaterun -state full"; + $fpaCommand .= " -updaterun -set_state full"; } $fpaCommand .= " -dbname $dbname" if defined $dbname; Index: /branches/eam_branches/20090820/ippScripts/scripts/detrend_process_imfile.pl =================================================================== --- /branches/eam_branches/20090820/ippScripts/scripts/detrend_process_imfile.pl (revision 25869) +++ /branches/eam_branches/20090820/ippScripts/scripts/detrend_process_imfile.pl (revision 25870) @@ -94,5 +94,5 @@ 'DOMEFLAT' => 'PPIMAGE.OUTPUT.DETREND', 'SKYFLAT' => 'PPIMAGE.OUTPUT.DETREND', - 'FRINGE' => undef, + 'FRINGE' => 'PPIMAGE.OUTPUT.DETREND', }; Index: /branches/eam_branches/20090820/ippScripts/scripts/dist_make_fileset.pl =================================================================== --- /branches/eam_branches/20090820/ippScripts/scripts/dist_make_fileset.pl (revision 25869) +++ /branches/eam_branches/20090820/ippScripts/scripts/dist_make_fileset.pl (revision 25870) @@ -40,5 +40,5 @@ # Parse the command-line arguments my ($dist_id, $dist_dir, $target_id, $stage, $stage_id, $dest_id, $product_name, $ds_dbhost, $ds_dbname); -my ($label, $filter); +my ($label, $dist_group, $filter); my ($dbname, $save_temps, $verbose, $no_update, $logfile); @@ -51,5 +51,6 @@ 'dest_id=s' => \$dest_id, # id for the product 'product_name=s' => \$product_name, # location of the data store directory for this product - 'label=s' => \$label, + 'label=s' => \$label, + 'dist_group=s' => \$dist_group, 'filter=s' => \$filter, 'ds_dbhost=s' => \$ds_dbhost, # database host for the datastore database @@ -63,5 +64,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --dist_id --dist_dir --target_id --stage --stage_id --label --filter --dest_id --ds_dbhost --ds_dbname", +pod2usage( -msg => "Required options: --dist_id --dist_dir --target_id --stage --stage_id --dist_group --filter --dest_id --ds_dbhost --ds_dbname", -exitval => 3) unless defined $dist_id and @@ -70,5 +71,5 @@ defined $stage and defined $stage_id and - defined $label and + defined $dist_group and defined $filter and defined $dest_id and @@ -161,5 +162,5 @@ $command .= " --ps0 $target_id --ps1 $stage --ps2 $stage_id --ps3 $prod_col_3"; - $command .= " --ps4 $label --ps5 $filter"; + $command .= " --ps4 $dist_group --ps5 $filter"; $command .= " --dbname $ds_dbname --dbhost $ds_dbhost"; Index: /branches/eam_branches/20090820/ippScripts/scripts/flatcorr_proc.pl =================================================================== --- /branches/eam_branches/20090820/ippScripts/scripts/flatcorr_proc.pl (revision 25869) +++ /branches/eam_branches/20090820/ippScripts/scripts/flatcorr_proc.pl (revision 25870) @@ -104,7 +104,8 @@ my ($DECs, $DECe) = split (",", $coords[1]); -# Run addstar -resort to ensure the db is indexed -{ - my $command = "$addstar -resort"; +# Run addstar -resort to ensure the db is indexed +# XXX addstar should be able to recognize and skip indexed tables +{ + my $command = "$addstar -resort"; $command .= " -D CATDIR $dvodb"; $command .= " -region $RAs $RAe $DECs $DECe"; Index: /branches/eam_branches/20090820/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- /branches/eam_branches/20090820/ippScripts/scripts/ipp_cleanup.pl (revision 25869) +++ /branches/eam_branches/20090820/ippScripts/scripts/ipp_cleanup.pl (revision 25870) @@ -91,6 +91,7 @@ # if there are no chipProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' + # XXX Why? This could just mean there's nothing to cleanup, or that we're trying to rerun an errored run. if (@$stdout_buf == 0) { - my $command = "$chiptool -chip_id $stage_id -updaterun -set_state new"; + my $command = "$chiptool -chip_id $stage_id -updaterun -set_state $error_state"; $command .= " -dbname $dbname" if defined $dbname; @@ -123,5 +124,4 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); -# if (!$config_file or ! -e $config_file) { unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for chipRun $stage_id $class_id " @@ -187,4 +187,15 @@ &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); } + + # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. + my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; + $command = "$magicdstool -dbname gpc1 -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); + } + } else { @@ -199,4 +210,16 @@ &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); } + + # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. + $command = "$chiptool -chip_id $stage_id -updaterun -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); + } + } } @@ -283,4 +306,15 @@ &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code); } + + # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. + my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; + $command = "$magicdstool -dbname gpc1 -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); + } + } else { # since 'camera' has only a single imfile, we can just update the run @@ -296,4 +330,6 @@ exit $PS_EXIT_UNKNOWN_ERROR; } + + exit 0; } @@ -314,4 +350,20 @@ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + # No skycells were found for some reason. + # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself." + my $command = "$warptool -updaterun -warp_id $stage_id -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); + } + + exit(0); } my $metadata = $mdcParser->parse(join "", @$stdout_buf) or @@ -389,4 +441,15 @@ &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); } + + # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. + my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; + $command = "$magicdstool -dbname gpc1 -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); + } + } else { my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state"; @@ -399,5 +462,6 @@ &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); } - exit $PS_EXIT_UNKNOWN_ERROR; + + # exit $PS_EXIT_UNKNOWN_ERROR; } } @@ -407,7 +471,5 @@ ### added for cleanup, based on warp stage entry if ($stage eq 'stack') { - die "--stage_id required for stage stack\n" if !$stage_id; - # this stage uses 'stacktool' my $stacktool = can_run('stacktool') or die "Can't find stacktool"; @@ -422,4 +484,21 @@ &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); } + + if (@$stdout_buf == 0) { + # No skycells were found for some reason. + # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself." + my $command = "$stacktool -updaterun -stack_id $stage_id -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); + } + + exit(0); + } + my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or &my_die("Unable to parse metadata config doc", "stack", $stage_id, $PS_EXIT_PROG_ERROR); @@ -434,22 +513,29 @@ my $status = 1; - if ($mode eq "goto_cleaned") { - my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); - - unless ($ipprc->file_exists($config_file)) { - print STDERR "skipping cleanup for stackRun $stage_id $skycell_id" . - " because config file is missing\n"; - $status = 0; - } - } - elsif ($mode eq "goto_scrubbed") { - my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); - - if ($ipprc->file_exists($config_file)) { - print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" . - " because config file is present\n"; - $status = 0; - } - } + if ((!exists($skyfile->{path_base}))|| + (!defined($path_base))) { + $status = 0; + } + if ($status) { + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); + + unless ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for stackRun $stage_id $skycell_id" . + " because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" . + " because config file is present\n"; + $status = 0; + } + } + } + if ($status) { my @files = (); @@ -475,11 +561,11 @@ my $command = "$stacktool -stack_id $stage_id"; if ($mode eq "goto_purged") { - $command .= " -updaterun -state purged"; + $command .= " -updaterun -set_state purged"; } elsif ($mode eq "goto_cleaned") { - $command .= " -updaterun -state cleaned"; + $command .= " -updaterun -set_state cleaned"; } elsif ($mode eq "goto_scrubbed") { - $command .= " -updaterun -state scrubbed"; + $command .= " -updaterun -set_state scrubbed"; } $command .= " -dbname $dbname" if defined $dbname; @@ -491,6 +577,17 @@ &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); } + + # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. + my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; + $command = "$magicdstool -dbname gpc1 -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); + } + } else { - my $command = "$stacktool -updaterun -stack_id $stage_id -state $error_state"; + my $command = "$stacktool -updaterun -stack_id $stage_id -set_state $error_state"; $command .= " -dbname $dbname" if defined $dbname; @@ -501,5 +598,5 @@ &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); } - exit $PS_EXIT_UNKNOWN_ERROR; +# exit $PS_EXIT_UNKNOWN_ERROR; } } @@ -523,4 +620,21 @@ &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); } + + if (@$stdout_buf == 0) { + # No skycells were found for some reason. + # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself." + my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); + } + + exit(0); + } + my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or &my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR); @@ -535,20 +649,27 @@ my $status = 1; - if ($mode eq "goto_cleaned") { - my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); - - unless ($ipprc->file_exists($config_file)) { - print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" . - " because config file ($config_file) is missing\n"; - $status = 0; - } - } - elsif ($mode eq "goto_scrubbed") { - my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); - - if ($ipprc->file_exists($config_file)) { - print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" . - " because config file ($config_file) is present\n"; - $status = 0; + if ((!exists($skyfile->{path_base}))|| + (!defined($path_base))) { + $status = 0; + } + if ($status) { + + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); + + unless ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" . + " because config file ($config_file) is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" . + " because config file ($config_file) is present\n"; + $status = 0; + } } } @@ -588,14 +709,14 @@ # print STDERR "MY STATUS: $status\n"; if ($status) { - my $command = "$difftool -diff_id $stage_id"; - - if ($mode eq "goto_purged") { - $command .= " -updaterun -state purged"; + my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id"; + + if ($mode eq "goto_purged") { + $command .= " -topurgedskyfile"; } elsif ($mode eq "goto_cleaned") { - $command .= " -updaterun -state cleaned"; + $command .= " -tocleanedskyfile"; } elsif ($mode eq "goto_scrubbed") { - $command .= " -updaterun -state scrubbed"; + $command .= " -toscrubbedskyfile"; } @@ -608,6 +729,17 @@ &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); } + + # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. + my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; + $command = "$magicdstool -dbname gpc1 -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); + } + } else { - my $command = "$difftool -updaterun -diff_id $stage_id -state $error_state"; + my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state"; $command .= " -dbname $dbname" if defined $dbname; @@ -619,5 +751,5 @@ &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); } - exit $PS_EXIT_UNKNOWN_ERROR; +# exit $PS_EXIT_UNKNOWN_ERROR; } } @@ -646,5 +778,5 @@ # if there are no fakeProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' if (@$stdout_buf == 0) { - my $command = "$faketool -fake_id $stage_id -updaterun -set_state new"; + my $command = "$faketool -fake_id $stage_id -updaterun -set_state $error_state"; $command .= " -dbname $dbname" if defined $dbname; Index: anches/eam_branches/20090820/ippScripts/scripts/magic_definerun.pl =================================================================== --- /branches/eam_branches/20090820/ippScripts/scripts/magic_definerun.pl (revision 25869) +++ (revision ) @@ -1,236 +1,0 @@ -#!/usr/bin/env perl - -# program to define a magic run -# The magictool mode will not queue a run until all diffs from an exposure are -# successfully completed. There may also be issues with multiple warp runs -# for a single exposure. -# This program selects the diffs based on the value for warp_id if provided - -use Carp; -use warnings; -use strict; - -## report the program and machine -use Sys::Hostname; -my $host = hostname(); - -use vars qw( $VERSION ); -$VERSION = '0.01'; - -use IPC::Cmd 0.36 qw( can_run run ); -use File::Temp qw( tempfile ); -use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::List qw( parse_md_list ); - -use PS::IPP::Config 1.01 qw( :standard ); - -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); - -# Look for programs we need -my $missing_tools; -my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1); -my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); -my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); -if ($missing_tools) { - warn("Can't find required tools."); - exit($PS_EXIT_CONFIG_ERROR); -} - -# Parse the command-line arguments -my ($exp_id, $warp_id, $min_diff_id, $label, $workdir, $dbname, $save_temps, $verbose); -GetOptions( - 'exp_id=s' => \$exp_id, # exposure identifier - 'warp_id=s' => \$warp_id, # warp identifier - 'min_diff_id=s' => \$min_diff_id, # minimum diff id to consider - 'label=s' => \$label, # label for images of interest - 'dbname=s' => \$dbname, # Database name - 'workdir=s' => \$workdir, # workdir - 'save-temps' => \$save_temps, # Save temporary files? - 'verbose' => \$verbose, # Print stuff? - ) or pod2usage( 2 ); - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --warp_id --workdir", - -exitval => 3) unless - defined $workdir and - defined $warp_id; - -my $ipprc = PS::IPP::Config->new() or my_die("Unable to set up", $PS_EXIT_CONFIG_ERROR); # IPP configuration - -my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - -if (!$exp_id) { - my $command = "$warptool -warped -warp_id $warp_id -limit 1"; - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform warptool -warped error_code: $error_code", $error_code); - } - - my $warptool_output = join "", @$stdout_buf; - - my $warpskyfiles = parse_md_fast($mdcParser, $warptool_output); - &my_die("Unable to parse metadata list", $PS_EXIT_UNKNOWN_ERROR) unless $warpskyfiles; - - my $warped = $warpskyfiles->[0]; - $exp_id = $warped->{exp_id}; - &my_die("failed to get exp_id for warp: $warp_id", $PS_EXIT_UNKNOWN_ERROR) unless $exp_id; -} - -### Get a list of inputs -my $inputs; # List of inputs -{ - my $command = "$difftool -diffskyfile -exp_id $exp_id"; - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool -diffskyfile error_code: $error_code", $error_code); - } - - my $difftool_output = join "", @$stdout_buf; - - $inputs = parse_md_fast($mdcParser, $difftool_output); - &my_die("Unable to parse metadata list", $PS_EXIT_UNKNOWN_ERROR) unless $inputs; -} - -my $num_skyfiles = @$inputs; -if (! $num_skyfiles) { - carp("failed to find any diffskyfiles for exposure $exp_id"); - exit(1); -} - -print "$num_skyfiles skyfiles found for $exp_id\n" if $verbose; - -my $diff_id_for_run; -if ($warp_id or $min_diff_id) { - # filter the inputs - my $i = 0; - while ($i < scalar @$inputs) { - my $keep = 1; - my $sf = $inputs->[$i]; - - if ($warp_id) { - if ($warp_id ne $sf->{warp1}) {; - $keep = 0; - } - } - if ($min_diff_id) { - if ($sf->{diff_id} < $min_diff_id) { - $keep = 0; - } - } - if ( $keep) { - $i++; - $diff_id_for_run = $sf->{diff_id} if !$diff_id_for_run; - } else { - # remove it - splice @$inputs, $i, 1; - } - } - $num_skyfiles = $i; - print "$num_skyfiles skyfiles accepted for $exp_id\n" if $verbose; -} - -if (!$num_skyfiles) { - # should I return an error here? - &my_die("no skyfiles matching criteria found for exp_id $exp_id", 0); -} - -my $magic_id; -{ - my $command = "$magictool -definerun -exp_id $exp_id -workdir $workdir -simple"; - $command .= " -diff_id $diff_id_for_run" if defined $diff_id_for_run; - $command .= " -label $label" if defined $label; - $command .= " -dbname $dbname" if defined $dbname; - - print "$command\n"; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool -diffskyfile: $error_code", $error_code); - } - - my $buffer = join "", @$stdout_buf; - ($magic_id, undef) = split " ", $buffer; -} - -&my_die("failed to retrieve new magic run information", $PS_EXIT_CONFIG_ERROR) if !$magic_id; - -### This is left over from when diffs were composed of a single skycell -### When we're not too busy, it should be deleted in favour of: -### magicRun JOIN diffSkyfile USING(diff_id) WHERE diffSkyfile.fault = 0 AND diffSkyfile.quality = 0 -foreach my $diff_skyfile (@$inputs) { - my $skycell_id = $diff_skyfile->{skycell_id}; - if ($diff_skyfile->{fault} > 0) { - print "skipping faulted diffSkyfile: $skycell_id\n" if $verbose; - next; - } - my $diff_id = $diff_skyfile->{diff_id}; - my $command = "$magictool -addinputskyfile"; - $command .= " -magic_id $magic_id"; - $command .= " -node $skycell_id"; - $command .= " -dbname $dbname" if $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magictool -addinputskyfile for magicRun: $magic_id skycell_id: $skycell_id: $error_code", $error_code); - } -} - -# set the new magicRun to state 'run' -{ - my $command = "$magictool -updaterun -magic_id $magic_id -state run"; - $command .= " -dbname $dbname" if $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magictool -updaterun for magicRun: $magic_id: $error_code", $error_code); - } -} - -sub my_die -{ - my $msg = shift; # Warning message on die - my $exit_code = shift; # Exit code to add - - carp($msg); - exit $exit_code; -} - -sub parse_md_fast { - my $mdcParser = shift; - my $input = shift; - my $output = (); - - my @whole = split /\n/, $input; - my @single = (); - - my $n; - while ( ($n = @whole) > 0) { - my $value = shift @whole; - push @single, $value; - if ($value =~ /^\s*END\s*$/) { - push @single, "\n"; - - my $list = parse_md_list( $mdcParser->parse( join("\n", @single ) ) ) or - print STDERR "Unable to parse metdata config doc" and return undef; - push @$output, $list->[0]; - - @single = (); - } - } - return $output; -} - -__END__ Index: /branches/eam_branches/20090820/ippScripts/scripts/magic_destreak_cleanup.pl =================================================================== --- /branches/eam_branches/20090820/ippScripts/scripts/magic_destreak_cleanup.pl (revision 25870) +++ /branches/eam_branches/20090820/ippScripts/scripts/magic_destreak_cleanup.pl (revision 25870) @@ -0,0 +1,267 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Temp qw( tempfile ); +use File::Basename qw( basename dirname ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config 1.01 qw( :standard ); +use Nebulous::Client; +use DBI; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Parse the command-line arguments +my ($magic_ds_id, $camera); +my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile); + +GetOptions( + 'magic_ds_id=s' => \$magic_ds_id,# Magic destreak run identifier + 'camera=s' => \$camera, # camera for evaluating file rules + 'save-temps' => \$save_temps, # Save temporary files? + 'dbname=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print stuff? + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --magic_ds_id --camera", + -exitval => 3) unless + defined $magic_ds_id and + defined $camera; +# defined $stage and +# defined $stage_id; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration +$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $PS_EXIT_SYS_ERROR ) if $logfile; + +if (0) { +my $nebulousServer = metadataLookupStr( $ipprc->{_siteConfig}, 'NEB_SERVER' ); +&my_die("cannot find NEB_SERVER in site configuration", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$nebulousServer; + +my $nebulous = eval { Nebulous::Client->new( proxy => $nebulousServer ); }; +if ($@ or not defined $nebulous) { + &my_die ("Unable to create a Nebulous::Client object with proxy $nebulousServer", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); +} +} + +$dbname = metadataLookupStr( $ipprc->{_siteConfig}, 'DBNAME' ) if !$dbname; +&my_die ("Unable to find DBNAME in site or command line arguments", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$dbname; + +my $dbuser = metadataLookupStr( $ipprc->{_siteConfig}, 'DBUSER' ); +&my_die ("Unable to find DBUSER in site", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$dbuser; + +my $dbpassword = metadataLookupStr( $ipprc->{_siteConfig}, 'DBPASSWORD' ); +&my_die ("Unable to find DBPASSWORD in site", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$dbpassword; + +my $dbserver = metadataLookupStr( $ipprc->{_siteConfig}, 'DBSERVER' ); +&my_die ("Unable to find DBSERVER in site", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$dbserver; + +my $dsn = "DBI:mysql:host=$dbserver;database=$dbname"; +my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) or die "Cannot connect to mysql server\n"; + +my $q1 = "SELECT magicDSRun.*, camera, camProcessedExp.path_base AS cam_path_base, magicRun.inverse" + . " FROM magicDSRun JOIN magicRun USING(magic_id) JOIN rawExp USING(exp_id) LEFT JOIN camProcessedExp USING(cam_id)" + . " WHERE magic_ds_id = $magic_ds_id"; +my $q2 = "SELECT * from magicDSFile where magic_ds_id = $magic_ds_id"; + +# my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +my $stmt1 = $dbh->prepare($q1); +$stmt1->execute(); +my $nrows = $stmt1->rows; +&my_die ("Unable to find magicDSRun $magic_ds_id", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$nrows; +my $run = $stmt1->fetchrow_hashref(); + +my $state = $run->{state}; +my $stage = $run->{stage}; +my $stage_id = $run->{stage_id}; +my $cam_path_base = $run->{cam_path_base}; +my $inverse = $run->{inverse}; + + +&my_die("unexpected run state found: $state", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $state ne "goto_cleaned"; +&my_die("clean not allowed for raw stage, use goto_restore", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $stage eq "raw"; + + +my $stmt2 = $dbh->prepare($q2); +$stmt2->execute(); + +my $num_components = 0; +while (my $comp = $stmt2->fetchrow_hashref()) { + $num_components++; + + my $component = $comp->{component}; + my $backup_path_base = $comp->{backup_path_base}; + my $recovery_path_base = $comp->{recovery_path_base}; + my ($bimage, $bmask, $bch_mask, $bweight, $bsources, $bastrom); + my ($rimage, $rmask, $rch_mask, $rweight, $rsources, $rastrom); + + if ($stage eq "chip") { + # we use the mask output from the camera stage for input and replace + # the output of the chip stage with that mask as well. + my $cammask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $component); + + if ($backup_path_base) { + $bimage = $ipprc->filename("PPIMAGE.CHIP", $backup_path_base, $component); + $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $component); + $bweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $backup_path_base, $component); + # This is kludgey but correct + + my $prefix; + if ($backup_path_base =~ /\.SR/) { + $prefix = ""; + } else { + $prefix = "SR_"; + } + + $bmask = dirname($backup_path_base) . "/" . $prefix . basename($cammask); + } + if ($recovery_path_base) { + $rimage = $ipprc->filename("PPIMAGE.CHIP", $recovery_path_base, $component); + $rch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $component); + $rweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $recovery_path_base, $component); + # This is kludgey but correct + $rmask = dirname($recovery_path_base) . "/" . basename($cammask); + } + } elsif ($stage eq "camera") { + if ($backup_path_base) { + $bastrom = $ipprc->filename("PSASTRO.OUTPUT", $backup_path_base); + } + } elsif ($stage eq "warp") { + if ($backup_path_base) { + $bimage = $ipprc->filename("PSWARP.OUTPUT", $backup_path_base); + $bmask = $ipprc->filename("PSWARP.OUTPUT.MASK", $backup_path_base); + $bweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $backup_path_base); + $bsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $backup_path_base); + } + if ($recovery_path_base) { + $rimage = $ipprc->filename("PSWARP.OUTPUT", $recovery_path_base); + $rmask = $ipprc->filename("PSWARP.OUTPUT.MASK", $recovery_path_base); + $rweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $recovery_path_base); + $rsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $recovery_path_base); + } + } elsif ($stage eq "diff") { + my $name = "PPSUB.OUTPUT"; + if ($backup_path_base) { + $bimage = $ipprc->filename($name, $backup_path_base); + $bmask = $ipprc->filename("$name.MASK", $backup_path_base); + $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base); + $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base); + } + if ($recovery_path_base) { + $rimage = $ipprc->filename($name, $recovery_path_base); + $rmask = $ipprc->filename("$name.MASK", $recovery_path_base); + $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base); + $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base); + } + } + + delete_files($rimage, $rmask, $rweight, $rsources, $rastrom, $bimage, $bmask, $bweight, $bsources, $bastrom); + + if ($stage eq "diff" and $inverse) { + my $name = "PPSUB.INVERSE"; + if ($backup_path_base) { + $bimage = $ipprc->filename($name, $backup_path_base); + $bmask = $ipprc->filename("$name.MASK", $backup_path_base); + $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base); + $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base); + } + if ($recovery_path_base) { + $rimage = $ipprc->filename($name, $recovery_path_base); + $rmask = $ipprc->filename("$name.MASK", $recovery_path_base); + $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base); + $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base); + } + delete_files($rimage, $rmask, $rweight, $rsources, undef, $bimage, $bmask, $bweight, $bsources, undef); + } +} + +if (!$no_update and ($num_components > 0)) { + my $result = $dbh->do("DELETE FROM magicDSFile WHERE magic_ds_id = ?", undef, $magic_ds_id); + # my $result = $stmt3->do($magic_ds_id); + my_die("attempt to delete magicDSFiles failed", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR) if $result eq "0E0"; +} + +if (!$no_update) { + my $result = $dbh->do("UPDATE magicDSRun SET state = 'cleaned' WHERE magic_ds_id = ?", undef, $magic_ds_id); + my_die("attempt to update magicDSRun failed", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR) if $result eq "0E0"; +} else { + print STDERR "skipping update of magicDSRun\n"; +} + + +### Pau. + +sub delete_files { + foreach my $file (@_) { + if ($file) { + if ($ipprc->file_exists($file)) { + if (!$no_update) { + print STDERR "deleting $file\n" if $verbose; + $ipprc->file_delete($file) or my_die("Failed to delete $file", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR); + } else { + print STDERR "skipping delete $file\n"; + } + } else { + print STDERR "$file not found\n" if $verbose; + } + } + } +} + + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $magic_ds_id = shift; # Magic DS identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + my $command = "$magicdstool -updaterun -fault $exit_code"; + $command .= " -magic_ds_id $magic_ds_id"; + $command .= " -dbname $dbname" if defined $dbname; + + # Add the processed file to the database + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + carp("failed to update database for $magic_ds_id"); + } + } else { + print "Skipping command: $command\n"; + } + + carp($msg); + exit $exit_code; +} + +__END__ Index: /branches/eam_branches/20090820/ippScripts/scripts/stack_skycell.pl =================================================================== --- /branches/eam_branches/20090820/ippScripts/scripts/stack_skycell.pl (revision 25869) +++ /branches/eam_branches/20090820/ippScripts/scripts/stack_skycell.pl (revision 25870) @@ -304,5 +304,5 @@ $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); } else { - $mode = "-updaterun -state full"; + $mode = "-updaterun -set_state full"; } $command .= " $mode -stack_id $stack_id"; Index: /branches/eam_branches/20090820/ippTasks/addstar.pro =================================================================== --- /branches/eam_branches/20090820/ippTasks/addstar.pro (revision 25869) +++ /branches/eam_branches/20090820/ippTasks/addstar.pro (revision 25870) @@ -121,10 +121,10 @@ book getword addPendingExp $pageName exp_tag -var EXP_TAG book getword addPendingExp $pageName add_id -var ADD_ID - book getword addPendingExp $pageName cam_id -var CAM_ID + book getword addPendingExp $pageName camroot -var CAMROOT book getword addPendingExp $pageName workdir -var WORKDIR_TEMPLATE + book getword addPendingExp $pageName reduction -var REDUCTION book getword addPendingExp $pageName dvodb -var DVODB + book getword addPendingExp $pageName image_only -var IMAGE_ONLY book getword addPendingExp $pageName dbname -var DBNAME - book getword addPendingExp $pageName reduction -var REDUCTION - book getword addPendingExp $pageName state -var RUN_STATE # specify choice of remote host based on camera and chip (class_id) @@ -144,16 +144,16 @@ ## generate outroot specific to this exposure (& chip) sprintf outroot "%s/%s/%s.add.%s" $WORKDIR $EXP_TAG $EXP_TAG $ADD_ID - sprintf camroot "%s/%s/%s.cm.%s" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID stdout $LOGDIR/addstar.exp.log stderr $LOGDIR/addstar.exp.log - $run = addstar_run.pl --exp_tag $EXP_TAG --add_id $ADD_ID --camera $CAMERA --outroot $outroot --camroot $camroot --redirect-output --run-state $RUN_STATE + $run = addstar_run.pl --exp_tag $EXP_TAG --add_id $ADD_ID --camera $CAMERA --dvodb $DVODB --camroot $CAMROOT --outroot $outroot --redirect-output if ("$REDUCTION" != "NULL") $run = $run --reduction $REDUCTION end - if ("$DVODB" != "NULL") - $run = $run --dvodb $DVODB - end + if ("$IMAGE_ONLY" == "T") + $run = $run --image-only + end + add_standard_args run Index: /branches/eam_branches/20090820/ippTasks/destreak.pro =================================================================== --- /branches/eam_branches/20090820/ippTasks/destreak.pro (revision 25869) +++ /branches/eam_branches/20090820/ippTasks/destreak.pro (revision 25870) @@ -10,4 +10,5 @@ book init magicToDS book init magicDSToRevert +book init magicDSToCleanup ### indexes into Database lists @@ -16,4 +17,5 @@ $magicDSToRevert_DB = 0 $magicDSCompletedRevert_DB = 0 +$magicDSToCleanup_DB = 0 #list of stages @@ -32,4 +34,5 @@ book listbook magicToDS book listbook magicDSToRevert + book init magicDSToCleanup end @@ -38,4 +41,5 @@ book init magicToDS book init magicDSToRevert + book init magicDSToCleanup end @@ -71,4 +75,13 @@ end +macro destreak.cleanup.on + task destreak.cleanup.load + active true + end + task destreak.cleanup.run + active true + end +end + ### Turn tasks off macro destreak.off @@ -92,4 +105,12 @@ end task destreak.completed.revert + active false + end +end +macro destreak.cleanup.off + task destreak.cleanup.load + active false + end + task destreak.cleanup.run active false end @@ -489,2 +510,125 @@ end +task destreak.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 20 + npending 1 + active false + + stdout NULL + stderr $LOGSUBDIR/destreak.cleanup.load.log + + task.exec + $run = magicdstool -tocleanup + if ($DB:n == 0) + option DEFAULT + else + + # save the DB name for the exit tasks + option $DB:$magicDSToCleanup_DB + $run = $run -dbname $DB:$magicDSToCleanup_DB + + # only bump database number after we have gone through all of the stages + $magicDSToCleanup_DB ++ + if ($magicDSToCleanup_DB >= $DB:n) set magicDSToCleanup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicDSToCleanup -key magic_ds_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicDSToCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicDSToCleanup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task destreak.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + stdout $LOGSUBDIR/destreak.cleanup.run.log + stderr $LOGSUBDIR/destreak.cleanup.run.log + + book npages magicDSToCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicDSToCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicDSToCleanup $pageName pantaskState RUN + book getword magicDSToCleanup $pageName magic_ds_id -var MAGIC_DS_ID + book getword magicDSToCleanup $pageName camera -var CAMERA + book getword magicDSToCleanup $pageName outroot -var OUTROOT + book getword magicDSToCleanup $pageName dbname -var DBNAME + + sprintf logfile "%s/mds.%s.cleanup.log" $OUTROOT $MAGIC_DS_ID + + host anyhost + + $run = magic_destreak_cleanup.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --logfile $logfile + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit 0 + process_exit magicDSToCleanup $options:0 $JOB_STATUS + end + + # locked list + task.exit default + showcommand failure + process_exit magicDSToCleanup $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword magicDSToCleanup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicDSToCleanup $options:0 pantaskState TIMEOUT + end +end + Index: /branches/eam_branches/20090820/ippTasks/flatcorr.pro =================================================================== --- /branches/eam_branches/20090820/ippTasks/flatcorr.pro (revision 25869) +++ /branches/eam_branches/20090820/ippTasks/flatcorr.pro (revision 25870) @@ -2,45 +2,4 @@ ## this file contains the tasks for running the flatcorr stage ## these tasks use the book 'flatcorrBook' - -# test for required global variables -check.globals - -book init flatcorrBook - -macro flatcorr.reset - book init flatcorrBook -end - -macro flatcorr.status - book listbook flatcorrBook -end - -macro flatcorr.on - task flatcorr.addcamera - active true - end - task flatcorr.load - active true - end - task flatcorr.run - active true - end -end - -macro flatcorr.off - task flatcorr.addcamera - active false - end - task flatcorr.load - active false - end - task flatcorr.run - active false - end -end - -# these variables will cycle through the known ippdb database names -$flatcorr_addcamera_DB = 0 -$flatcorr_pendingprocess_DB = 0 # a flat-field correction run is initiated (manually) with a command like: @@ -55,7 +14,11 @@ # as the chip analysis progresses, we need to occasionally migrate the # completed chips to the camera stage by calling: -# flatcorr -addcamera - -# we wait for the completed chip and camera analysis and search for +# flatcorr -advancecamera + +# as the camera analysis progresses, we need to occasionally migrate the +# completed exposures to the addstar stage by calling: +# flatcorr -advanceaddstar + +# we wait for the completed addstar jobs and search for # completed processing analysis with: # flatcorr -pendingprocess @@ -67,6 +30,54 @@ # flatcorr -addprocess -fault N +# test for required global variables +check.globals + +book init flatcorrBook + +macro flatcorr.reset + book init flatcorrBook +end + +macro flatcorr.status + book listbook flatcorrBook +end + +macro flatcorr.on + task flatcorr.advancecamera + active true + end + task flatcorr.advanceaddstar + active true + end + task flatcorr.load + active true + end + task flatcorr.run + active true + end +end + +macro flatcorr.off + task flatcorr.advancecamera + active false + end + task flatcorr.advanceaddstar + active false + end + task flatcorr.load + active false + end + task flatcorr.run + active false + end +end + +# these variables will cycle through the known ippdb database names +$flatcorr_advancecamera_DB = 0 +$flatcorr_advanceaddstar_DB = 0 +$flatcorr_pendingprocess_DB = 0 + # migrate complete flatcorr chips to the camera stage analysis -task flatcorr.addcamera +task flatcorr.advancecamera host local @@ -80,12 +91,59 @@ # define the command (does not depend on previous queries) if ($DB:n == 0) - command flatcorr -addcamera + command flatcorr -advancecamera else # save the DB name for the exit tasks # note that this DB name refers to the ippdb, not the dvodb - option $DB:$flatcorr_addcamera_DB - command flatcorr -addcamera -dbname $DB:$flatcorr_addcamera_DB - $flatcorr_addcamera_DB ++ - if ($flatcorr_addcamera_DB >= $DB:n) set flatcorr_addcamera_DB = 0 + option $DB:$flatcorr_advancecamera_DB + command flatcorr -advancecamera -dbname $DB:$flatcorr_advancecamera_DB + $flatcorr_advancecamera_DB ++ + if ($flatcorr_advancecamera_DB >= $DB:n) set flatcorr_advancecamera_DB = 0 + end + end + + # silently drop stdout + stdout NULL + stderr $LOGDIR/flatcorr.log + + # success (no action required) + task.exit $EXIT_SUCCESS + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# migrate complete flatcorr exposures to the addstar process +task flatcorr.advanceaddstar + host local + + # check the list of pending flatcorr runs + periods -poll 5 + periods -exec 30 + periods -timeout 60 + npending 1 + + task.exec + # define the command (does not depend on previous queries) + if ($DB:n == 0) + command flatcorr -advanceaddstar + else + # save the DB name for the exit tasks + # note that this DB name refers to the ippdb, not the dvodb + option $DB:$flatcorr_advanceaddstar_DB + command flatcorr -advanceaddstar -dbname $DB:$flatcorr_advanceaddstar_DB + $flatcorr_advanceaddstar_DB ++ + if ($flatcorr_advanceaddstar_DB >= $DB:n) set flatcorr_advanceaddstar_DB = 0 end end Index: /branches/eam_branches/20090820/ippTasks/ipphosts.mhpcc.config =================================================================== --- /branches/eam_branches/20090820/ippTasks/ipphosts.mhpcc.config (revision 25869) +++ /branches/eam_branches/20090820/ippTasks/ipphosts.mhpcc.config (revision 25870) @@ -12,5 +12,6 @@ sky06 STR ipp012 sky07 STR ipp013 - sky08 STR ipp014 +# sky08 STR ipp014 + sky08 STR ipp047 sky09 STR ipp015 sky10 STR ipp016 @@ -55,6 +56,8 @@ camera STR GPC1 - XY01 STR ipp014 - XY02 STR ipp014 +# XY01 STR ipp014 +# XY02 STR ipp014 + XY01 STR ipp047 + XY02 STR ipp047 XY03 STR ipp038 XY04 STR ipp038 @@ -134,6 +137,8 @@ camera STR gpc1 - ota01 STR ipp014 - ota02 STR ipp014 +# ota01 STR ipp014 +# ota02 STR ipp014 + ota01 STR ipp047 + ota02 STR ipp047 ota03 STR ipp038 ota04 STR ipp038 Index: /branches/eam_branches/20090820/ippTasks/pstamp.pro =================================================================== --- /branches/eam_branches/20090820/ippTasks/pstamp.pro (revision 25869) +++ /branches/eam_branches/20090820/ippTasks/pstamp.pro (revision 25870) @@ -111,5 +111,5 @@ task.exec - stdout $LOGSUBDIR/pstamp.request.find.log + stdout NULL stderr $LOGSUBDIR/pstamp.request.find.log if ($DB:n == 0) Index: /branches/eam_branches/20090820/ippTasks/rcserver.pro =================================================================== --- /branches/eam_branches/20090820/ippTasks/rcserver.pro (revision 25869) +++ /branches/eam_branches/20090820/ippTasks/rcserver.pro (revision 25870) @@ -129,5 +129,5 @@ book getword rcPendingFS $pageName stage -var STAGE book getword rcPendingFS $pageName stage_id -var STAGE_ID - book getword rcPendingFS $pageName label -var LABEL + book getword rcPendingFS $pageName dist_group -var DIST_GROUP book getword rcPendingFS $pageName filter -var FILTER book getword rcPendingFS $pageName dist_dir -var DIST_DIR @@ -149,5 +149,5 @@ book setword rcPendingFS $pageName pantaskState RUN - $run = dist_make_fileset.pl --dist_id $DIST_ID --target_id $TARGET_ID --stage $STAGE --stage_id $STAGE_ID --label $LABEL --filter $FILTER --dest_id $DEST_ID --product_name $PRODUCT_NAME --ds_dbhost $DS_DBHOST --ds_dbname $DS_DBNAME --dist_dir $DIST_DIR + $run = dist_make_fileset.pl --dist_id $DIST_ID --target_id $TARGET_ID --stage $STAGE --stage_id $STAGE_ID --dist_group $DIST_GROUP --filter $FILTER --dest_id $DEST_ID --product_name $PRODUCT_NAME --ds_dbhost $DS_DBHOST --ds_dbname $DS_DBNAME --dist_dir $DIST_DIR add_standard_args run Index: /branches/eam_branches/20090820/ippTasks/simtest.basic.auto =================================================================== --- /branches/eam_branches/20090820/ippTasks/simtest.basic.auto (revision 25869) +++ /branches/eam_branches/20090820/ippTasks/simtest.basic.auto (revision 25870) @@ -47,5 +47,5 @@ name STR OBJECT-i check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter i -det_type FLAT" - launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc -set_data_group simtest_i -set_dist_group simtest" block STR NONE END @@ -54,5 +54,5 @@ name STR OBJECT-r check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLAT" - launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter r -label wait -set_label proc" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter r -label wait -set_label proc -set_data_group simtest_r -set_dist_group simtest" block STR NONE END @@ -61,5 +61,5 @@ automate METADATA name STR STACK - regular STR "stacktool -definebyquery -all -label proc -workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@" + regular STR "stacktool -definebyquery -set_label proc -set_data_group simtest_r -set_dist_group simtest -set_workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@" END @@ -67,5 +67,5 @@ automate METADATA name STR DIFF - regular STR "difftool -definewarpstack -label proc -workdir file://@CWD@/diff -good_frac 0.2 -dbname @DBNAME@" + regular STR "difftool -definewarpstack -set_label proc -set_workdir file://@CWD@/diff -good_frac 0.2 -dbname @DBNAME@" END Index: /branches/eam_branches/20090820/ippTasks/simtest.stack.auto =================================================================== --- /branches/eam_branches/20090820/ippTasks/simtest.stack.auto (revision 25869) +++ /branches/eam_branches/20090820/ippTasks/simtest.stack.auto (revision 25870) @@ -61,5 +61,5 @@ automate METADATA name STR STACK - regular STR "stacktool -definebyquery -all -label proc -workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@" + regular STR "stacktool -definebyquery -label proc -workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@" END Index: /branches/eam_branches/20090820/ippTasks/survey.pro =================================================================== --- /branches/eam_branches/20090820/ippTasks/survey.pro (revision 25869) +++ /branches/eam_branches/20090820/ippTasks/survey.pro (revision 25870) @@ -4,6 +4,12 @@ check.globals -book create SURVEY_DIFF -book create SURVEY_MAGIC +book getbook SURVEY_DIFF -var isBook +if ("$isBook" == "NULL") + book create SURVEY_DIFF +end +book getbook SURVEY_MAGIC -var isBook +if ("$isBook" == "NULL") + book create SURVEY_MAGIC +end $SURVEY_DIFF_DB = 0 Index: /branches/eam_branches/20090820/ippTools/configure.ac =================================================================== --- /branches/eam_branches/20090820/ippTools/configure.ac (revision 25869) +++ /branches/eam_branches/20090820/ippTools/configure.ac (revision 25870) @@ -1,5 +1,5 @@ AC_PREREQ(2.61) -AC_INIT([ipptools], [1.1.37], [ipp-support@ifa.hawaii.edu]) +AC_INIT([ipptools], [1.1.56], [ipp-support@ifa.hawaii.edu]) AC_CONFIG_SRCDIR([autogen.sh]) @@ -18,5 +18,5 @@ PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0]) PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) -PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.55]) +PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.56]) PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}" @@ -58,4 +58,7 @@ CFLAGS="${CFLAGS=} -Wall -Werror" +IPPDB_VERSION=`$PKG_CONFIG --modversion ippdb` +AC_DEFINE_UNQUOTED(IPPDB_VERSION, $IPPDB_VERSION, [Version of ippdb]) + AC_CONFIG_FILES([ Makefile Index: /branches/eam_branches/20090820/ippTools/share/Makefile.am =================================================================== --- /branches/eam_branches/20090820/ippTools/share/Makefile.am (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/Makefile.am (revision 25870) @@ -4,5 +4,10 @@ dist_pkgdata_DATA = \ + addtool_donecleanup.sql \ + addtool_find_cam_id.sql \ addtool_find_pendingexp.sql \ + addtool_find_processedexp.sql \ + addtool_pendingcleanupexp.sql \ + addtool_pendingcleanuprun.sql \ addtool_queue_cam_id.sql \ addtool_reset_faulted_runs.sql \ @@ -31,4 +36,5 @@ chiptool_revertprocessedimfile.sql \ chiptool_run.sql \ + chiptool_runstate.sql \ chiptool_export_imfile.sql \ chiptool_export_processed_imfile.sql \ @@ -84,4 +90,6 @@ dettool_toresidimfile.sql \ dettool_tostacked.sql \ + difftool_change_skyfile_data_state.sql \ + difftool_change_run_state.sql \ difftool_completed_runs.sql \ difftool_definewarpstack_part1.sql \ @@ -145,4 +153,5 @@ faketool_unmasked.sql \ flatcorr_chiprundone.sql \ + flatcorr_camerarundone.sql \ flatcorr_pendingprocess.sql \ flatcorr_inputimfile.sql \ @@ -182,4 +191,5 @@ magicdstool_getskycells.sql \ magicdstool_revertdestreakedfile.sql \ + magicdstool_tocleanup.sql \ magicdstool_todestreak_camera.sql \ magicdstool_todestreak_chip.sql \ @@ -196,4 +206,5 @@ pstamptool_datastore.sql \ pstamptool_listjob.sql \ + pstamptool_pendingdependent.sql \ pstamptool_pendingjob.sql \ pstamptool_pendingreq.sql \ @@ -205,4 +216,5 @@ pxadmin_create_mirror_tables.sql \ pxadmin_drop_tables.sql \ + pxadmin_update_version.sql \ pubtool_definerun.sql \ pubtool_pending.sql \ @@ -263,4 +275,5 @@ warptool_revertoverlap.sql \ warptool_revertwarped_delete.sql \ + warptool_runstate.sql \ warptool_scmap.sql \ warptool_tooverlap.sql \ Index: /branches/eam_branches/20090820/ippTools/share/addtool_donecleanup.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/addtool_donecleanup.sql (revision 25870) +++ /branches/eam_branches/20090820/ippTools/share/addtool_donecleanup.sql (revision 25870) @@ -0,0 +1,5 @@ +SELECT + addRun.* +FROM addRun +WHERE + addRun.state = 'cleaned' Index: /branches/eam_branches/20090820/ippTools/share/addtool_find_cam_id.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/addtool_find_cam_id.sql (revision 25870) +++ /branches/eam_branches/20090820/ippTools/share/addtool_find_cam_id.sql (revision 25870) @@ -0,0 +1,9 @@ +SELECT + camRun.* +FROM camRun +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) +WHERE + camRun.state = 'full' Index: /branches/eam_branches/20090820/ippTools/share/addtool_find_pendingexp.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/addtool_find_pendingexp.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/addtool_find_pendingexp.sql (revision 25870) @@ -1,4 +1,5 @@ SELECT addRun.*, + camProcessedExp.path_base as camroot, rawExp.exp_tag, rawExp.exp_id, @@ -10,10 +11,10 @@ JOIN camRun USING(cam_id) +JOIN camProcessedExp + USING(cam_id) JOIN chipRun USING(chip_id) -JOIN chipProcessedImfile - USING(chip_id) JOIN rawExp - ON chipRun.exp_id = rawExp.exp_id + USING(exp_id) LEFT JOIN addProcessedExp USING(add_id) @@ -23,4 +24,6 @@ camRun.state = 'full' AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL AND addMask.label IS NULL Index: /branches/eam_branches/20090820/ippTools/share/addtool_find_processedexp.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/addtool_find_processedexp.sql (revision 25870) +++ /branches/eam_branches/20090820/ippTools/share/addtool_find_processedexp.sql (revision 25870) @@ -0,0 +1,10 @@ +SELECT + addProcessedExp.*, + addRun.workdir +FROM addRun +JOIN camProcessedExp + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id Index: /branches/eam_branches/20090820/ippTools/share/addtool_pendingcleanupexp.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/addtool_pendingcleanupexp.sql (revision 25870) +++ /branches/eam_branches/20090820/ippTools/share/addtool_pendingcleanupexp.sql (revision 25870) @@ -0,0 +1,12 @@ +SELECT + addProcessedExp.*, + addRun.state, + addRun.workdir, + addRun.label, + addRun.dvodb, +FROM addRun +JOIN addProcessedExp + USING(add_id) +WHERE + (addRun.state = 'goto_cleaned' OR addRun.state = 'goto_purged' OR addRun.state = 'goto_scrubbed') + Index: /branches/eam_branches/20090820/ippTools/share/addtool_pendingcleanuprun.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/addtool_pendingcleanuprun.sql (revision 25870) +++ /branches/eam_branches/20090820/ippTools/share/addtool_pendingcleanuprun.sql (revision 25870) @@ -0,0 +1,13 @@ +SELECT + addRun.add_id, + rawExp.camera, + addRun.state +FROM addRun +JOIN camRun +USING (cam_id) +JOIN chipRun +USING (chip_id) +JOIN rawExp +USING (exp_id) +WHERE + (addRun.state = 'goto_cleaned' OR addRun.state = 'goto_purged' OR addRun.state = 'goto_scrubbed') Index: /branches/eam_branches/20090820/ippTools/share/addtool_queue_cam_id.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/addtool_queue_cam_id.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/addtool_queue_cam_id.sql (revision 25870) @@ -6,7 +6,10 @@ '%s', -- workdir '%s', -- workdir_state + '%s', -- reduction '%s', -- label + '%s', -- data_group '%s', -- dvodb - 0 -- magicked + '%s', -- note + %d -- image_only FROM camRun WHERE Index: /branches/eam_branches/20090820/ippTools/share/camtool_find_processedexp.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/camtool_find_processedexp.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/camtool_find_processedexp.sql (revision 25870) @@ -2,4 +2,5 @@ camProcessedExp.*, camRun.workdir, + camRun.label, chipRun.chip_id, rawExp.exp_tag, Index: /branches/eam_branches/20090820/ippTools/share/camtool_queue_chip_id.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/camtool_queue_chip_id.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/camtool_queue_chip_id.sql (revision 25870) @@ -9,4 +9,6 @@ '%s', -- workdir_state '%s', -- label + '%s', -- data_group + '%s', -- dist_group '%s', -- reduction '%s', -- expgroup @@ -14,5 +16,6 @@ '%s', -- tess_id '%s', -- end_stage - 0 -- magicked + 0, -- magicked + '%s' -- note FROM chipRun WHERE Index: /branches/eam_branches/20090820/ippTools/share/chiptool_completely_processed_exp.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/chiptool_completely_processed_exp.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/chiptool_completely_processed_exp.sql (revision 25870) @@ -7,4 +7,6 @@ workdir_state, label, + data_group, + dist_group, reduction, expgroup, @@ -12,5 +14,6 @@ tess_id, end_stage, - all_files_magicked as magicked + all_files_magicked as magicked, + note FROM (SELECT Index: /branches/eam_branches/20090820/ippTools/share/chiptool_processedimfile.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/chiptool_processedimfile.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/chiptool_processedimfile.sql (revision 25870) @@ -12,4 +12,5 @@ chipRun.state, chipRun.workdir, + chipRun.label, rawExp.exp_id, rawExp.exp_tag, Index: /branches/eam_branches/20090820/ippTools/share/chiptool_run.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/chiptool_run.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/chiptool_run.sql (revision 25870) @@ -2,2 +2,4 @@ chipRun.* FROM chipRun +JOIN rawExp + USING(exp_id) Index: /branches/eam_branches/20090820/ippTools/share/chiptool_runstate.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/chiptool_runstate.sql (revision 25870) +++ /branches/eam_branches/20090820/ippTools/share/chiptool_runstate.sql (revision 25870) @@ -0,0 +1,10 @@ +SELECT + chip_id, + chipRun.state, + exp_id, + magic_ds_id, + magicDSRun.state as magic_ds_state +FROM chipRun + JOIN rawExp USING(exp_id) + LEFT JOIN magicDSRun on stage = 'chip' and stage_id = chip_id and re_place + Index: /branches/eam_branches/20090820/ippTools/share/difftool_change_run_state.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/difftool_change_run_state.sql (revision 25870) +++ /branches/eam_branches/20090820/ippTools/share/difftool_change_run_state.sql (revision 25870) @@ -0,0 +1,15 @@ +-- change state of ddiffRun from goto_cleaned to cleaned or goto_purged to purged +-- when all of the consituant skyfiles are in the end state +-- arguments are new state (cleaned or purged) diff_id and new state again for +-- the chipProcessedImfile sub query +UPDATE diffRun + SET state = '%s' + WHERE + diffRun.diff_id = %lld + AND (SELECT + COUNT(diff_id) + FROM diffSkyfile + WHERE + diffSkyfile.diff_id = diffRun.diff_id + AND data_state != '%s' + ) = 0 Index: /branches/eam_branches/20090820/ippTools/share/difftool_change_skyfile_data_state.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/difftool_change_skyfile_data_state.sql (revision 25870) +++ /branches/eam_branches/20090820/ippTools/share/difftool_change_skyfile_data_state.sql (revision 25870) @@ -0,0 +1,9 @@ +-- handle changes in diffSkyfile.data_state. +-- Used for the modes tocleanedskyfile and topurgedskyfile +-- args are new data_state, diff_id, skycell_id and current expected state for diffRun +UPDATE diffSkyfile + SET + data_state = '%s' +WHERE + diff_id = %lld + AND skycell_id = '%s' Index: /branches/eam_branches/20090820/ippTools/share/difftool_pendingcleanuprun.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/difftool_pendingcleanuprun.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/difftool_pendingcleanuprun.sql (revision 25870) @@ -1,25 +1,112 @@ -- does this result in too many entries (one for each diffInputSkyfile?) -- all of this is just to get the camera used for the diff run -SELECT - diffRun.diff_id, - rawExp.camera, - diffRun.state -FROM diffRun -JOIN diffInputSkyfile - USING(diff_id) -JOIN warpSkyfile - ON diffInputSkyfile.warp1 = warpSkyfile.warp_id - AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id - AND diffInputSkyfile.tess_id = warpSkyfile.tess_id -JOIN warpRun - ON warpRun.warp_id = warpSkyfile.warp_id -JOIN fakeRun - USING(fake_id) -JOIN camRun - USING(cam_id) -JOIN chipRun - USING(chip_id) -JOIN rawExp - USING(exp_id) -WHERE - (diffRun.state = 'goto_cleaned' OR diffRun.state = 'goto_scrubbed' OR diffRun.state = 'goto_purged') +SELECT DISTINCT * FROM + (SELECT + diffRun.diff_id, + rawExp.camera, + diffRun.state, + diffRun.label + FROM diffRun + JOIN diffInputSkyfile + USING(diff_id) + JOIN warpSkyfile + ON diffInputSkyfile.warp1 = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.tess_id = warpSkyfile.tess_id + JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + WHERE + (diffRun.state = 'goto_cleaned' OR + diffRun.state = 'goto_scrubbed' OR + diffRun.state = 'goto_purged') + UNION + SELECT + diffRun.diff_id, + rawExp.camera, + diffRun.state, + diffRun.label + FROM diffRun + JOIN diffInputSkyfile + USING(diff_id) + JOIN warpSkyfile + ON diffInputSkyfile.warp2 = warpSkyfile.warp_id + AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id + AND diffInputSkyfile.tess_id = warpSkyfile.tess_id + JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + WHERE + (diffRun.state = 'goto_cleaned' OR + diffRun.state = 'goto_scrubbed' OR + diffRun.state = 'goto_purged') + UNION + SELECT + diffRun.diff_id, + rawExp.camera, + diffRun.state, + diffRun.label + FROM diffRun + JOIN diffInputSkyfile + USING(diff_id) + JOIN stackSumSkyfile + ON diffInputSkyfile.stack1 = stackSumSkyfile.stack_id + JOIN stackInputSkyfile + ON diffInputSkyfile.stack1 = stackInputSkyfile.stack_id + JOIN warpRun + ON warpRun.warp_id = stackInputSkyfile.warp_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + WHERE + (diffRun.state = 'goto_cleaned' OR + diffRun.state = 'goto_scrubbed' OR + diffRun.state = 'goto_purged') + UNION + SELECT + diffRun.diff_id, + rawExp.camera, + diffRun.state, + diffRun.label + FROM diffRun + JOIN diffInputSkyfile + USING(diff_id) + JOIN stackSumSkyfile + ON diffInputSkyfile.stack2 = stackSumSkyfile.stack_id + JOIN stackInputSkyfile + ON diffInputSkyfile.stack2 = stackInputSkyfile.stack_id + JOIN warpRun + ON warpRun.warp_id = stackInputSkyfile.warp_id + JOIN fakeRun + USING(fake_id) + JOIN camRun + USING(cam_id) + JOIN chipRun + USING(chip_id) + JOIN rawExp + USING(exp_id) + WHERE + (diffRun.state = 'goto_cleaned' OR + diffRun.state = 'goto_scrubbed' OR + diffRun.state = 'goto_purged') + ) as Foo + WHERE 1 Index: /branches/eam_branches/20090820/ippTools/share/difftool_skyfile.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/difftool_skyfile.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/difftool_skyfile.sql (revision 25870) @@ -5,4 +5,5 @@ diffRun.state, diffRun.workdir, + diffRun.label, diffRun.bothways, warp1, Index: /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_camera.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_camera.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_camera.sql (revision 25870) @@ -4,5 +4,6 @@ rawExp.exp_name as run_tag, camRun.magicked, - distTarget.label, + camRun.label, + distTarget.dist_group, distTarget.target_id, distTarget.clean @@ -12,5 +13,5 @@ JOIN distTarget ON distTarget.stage = 'camera' AND rawExp.filter = distTarget.filter - AND camRun.label = distTarget.label + AND camRun.dist_group = distTarget.dist_group JOIN rcInterest USING(target_id) LEFT JOIN distRun ON distRun.stage = 'camera' Index: /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_chip.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_chip.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_chip.sql (revision 25870) @@ -5,4 +5,5 @@ rawExp.exp_name as run_tag, chipRun.label, + chipRun.dist_group, distTarget.target_id, distTarget.clean @@ -10,5 +11,5 @@ JOIN rawExp USING(exp_id) JOIN distTarget ON distTarget.stage = 'chip' - AND chipRun.label = distTarget.label + AND chipRun.dist_group = distTarget.dist_group AND rawExp.filter = distTarget.filter JOIN rcInterest USING(target_id) Index: /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_diff.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_diff.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_diff.sql (revision 25870) @@ -4,5 +4,6 @@ diffRun.magicked, rawExp.exp_name as run_tag, - distTarget.label, + diffRun.label, + distTarget.dist_group, distTarget.target_id, distTarget.clean @@ -15,5 +16,5 @@ JOIN rawExp USING(exp_id) JOIN distTarget ON distTarget.stage = 'diff' - AND diffRun.label = distTarget.label + AND diffRun.dist_group = distTarget.dist_group AND distTarget.filter = rawExp.filter JOIN rcInterest USING(target_id) Index: /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_fake.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_fake.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_fake.sql (revision 25870) @@ -4,5 +4,6 @@ CAST(0 AS SIGNED) AS magicked, rawExp.exp_name as run_tag, - distTarget.label, + fakeRun.label, + distTarget.dist_group, distTarget.target_id, distTarget.clean @@ -11,5 +12,5 @@ JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) -JOIN distTarget ON distTarget.stage = 'fake' AND fakeRun.label = distTarget.label +JOIN distTarget ON distTarget.stage = 'fake' AND fakeRun.dist_group = distTarget.dist_group AND rawExp.filter = distTarget.filter JOIN rcInterest USING(target_id) Index: /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_raw.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_raw.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_raw.sql (revision 25870) @@ -4,10 +4,11 @@ rawExp.exp_name AS run_tag, rawExp.magicked, - distTarget.label, + chipRun.label, + distTarget.dist_group, distTarget.target_id, distTarget.clean FROM rawExp JOIN chipRun USING(exp_id) -JOIN distTarget ON distTarget.label = chipRun.label AND distTarget.stage = 'raw' +JOIN distTarget ON distTarget.dist_group = chipRun.dist_group AND distTarget.stage = 'raw' AND rawExp.filter = distTarget.filter JOIN rcInterest USING(target_id) Index: /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_stack.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_stack.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_stack.sql (revision 25870) @@ -5,5 +5,6 @@ -- run tag in the form 'stack.$skycell_id.$stack_id' CONCAT_WS('.', 'stack', stackRun.skycell_id, convert(stackRun.stack_id, CHAR)) as run_tag, - distTarget.label, + stackRun.label, + distTarget.dist_group, distTarget.target_id, distTarget.clean @@ -11,5 +12,5 @@ JOIN stackSumSkyfile USING(stack_id) JOIN distTarget ON distTarget.stage = 'stack' - AND stackRun.label = distTarget.label + AND stackRun.dist_group = distTarget.dist_group AND stackRun.filter = distTarget.filter JOIN rcInterest USING(target_id) Index: /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_warp.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_warp.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_warp.sql (revision 25870) @@ -4,5 +4,6 @@ warpRun.magicked, rawExp.exp_name as run_tag, - distTarget.label, + warpRun.label, + distTarget.dist_group, distTarget.target_id, distTarget.clean @@ -13,5 +14,5 @@ JOIN rawExp USING(exp_id) JOIN distTarget ON distTarget.stage = 'warp' - AND warpRun.label = distTarget.label + AND warpRun.dist_group = distTarget.dist_group AND rawExp.filter = distTarget.filter JOIN rcInterest USING(target_id) Index: /branches/eam_branches/20090820/ippTools/share/disttool_defineinterest.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/disttool_defineinterest.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/disttool_defineinterest.sql (revision 25870) @@ -1,4 +1,4 @@ INSERT INTO rcInterest - SELECT 0, dest_id, target_id, '@STATE@' + SELECT 0, rcDestination.dest_id, distTarget.target_id, '@STATE@' FROM distTarget JOIN rcDestination Index: /branches/eam_branches/20090820/ippTools/share/disttool_pending_camera.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/disttool_pending_camera.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/disttool_pending_camera.sql (revision 25870) @@ -2,4 +2,5 @@ distRun.dist_id, distRun.label, + distTarget.dist_group, stage, stage_id, @@ -16,4 +17,5 @@ chipRun.magicked FROM distRun +JOIN distTarget USING(target_id, stage, clean) JOIN camRun ON camRun.cam_id = distRun.stage_id JOIN camProcessedExp USING(cam_id) Index: /branches/eam_branches/20090820/ippTools/share/disttool_pending_chip.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/disttool_pending_chip.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/disttool_pending_chip.sql (revision 25870) @@ -2,8 +2,9 @@ distRun.dist_id, distRun.label, + distTarget.dist_group, stage, stage_id, chipProcessedImfile.class_id AS component, - clean, + distRun.clean, rawExp.camera, CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, @@ -16,4 +17,5 @@ chipProcessedImfile.magicked FROM distRun +JOIN distTarget USING(target_id, stage, clean) JOIN chipRun ON chipRun.chip_id = distRun.stage_id JOIN rawExp using(exp_id) Index: /branches/eam_branches/20090820/ippTools/share/disttool_pending_diff.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/disttool_pending_diff.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/disttool_pending_diff.sql (revision 25870) @@ -2,4 +2,5 @@ distRun.dist_id, distRun.label, + distTarget.dist_group, stage, stage_id, @@ -18,4 +19,5 @@ diffSkyfile.magicked FROM distRun +JOIN distTarget USING(target_id, stage, clean) JOIN diffRun ON stage_id = diff_id JOIN diffSkyfile using(diff_id) Index: /branches/eam_branches/20090820/ippTools/share/disttool_pending_fake.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/disttool_pending_fake.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/disttool_pending_fake.sql (revision 25870) @@ -2,4 +2,5 @@ distRun.dist_id, distRun.label, + distTarget.dist_group, stage, stage_id, @@ -16,4 +17,5 @@ 0 FROM distRun +JOIN distTarget USING(target_id, stage, clean) JOIN fakeRun ON fakeRun.fake_id = distRun.stage_id JOIN fakeProcessedImfile USING(fake_id) Index: /branches/eam_branches/20090820/ippTools/share/disttool_pending_raw.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/disttool_pending_raw.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/disttool_pending_raw.sql (revision 25870) @@ -2,4 +2,5 @@ distRun.dist_id, distRun.label, + distTarget.dist_group, stage, stage_id, @@ -16,4 +17,5 @@ rawImfile.magicked FROM distRun +JOIN distTarget USING(target_id, stage, clean) JOIN rawExp ON rawExp.exp_id = stage_id JOIN rawImfile USING(exp_id) @@ -35,4 +37,5 @@ distRun.dist_id, distRun.label, + distTarget.dist_group, stage, stage_id, @@ -49,4 +52,5 @@ rawExp.magicked FROM distRun +JOIN distTarget USING(target_id, stage, clean) JOIN rawExp ON exp_id = stage_id LEFT JOIN distComponent Index: /branches/eam_branches/20090820/ippTools/share/disttool_pending_stack.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/disttool_pending_stack.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/disttool_pending_stack.sql (revision 25870) @@ -2,4 +2,5 @@ distRun.dist_id, distRun.label, + distTarget.dist_group, stage, stage_id, @@ -18,4 +19,5 @@ 0 AS magicked FROM distRun +JOIN distTarget USING(target_id, stage, clean) JOIN stackRun ON stage_id = stack_id Index: /branches/eam_branches/20090820/ippTools/share/disttool_pending_warp.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/disttool_pending_warp.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/disttool_pending_warp.sql (revision 25870) @@ -2,4 +2,5 @@ distRun.dist_id, distRun.label, + distTarget.dist_group, stage, stage_id, @@ -16,4 +17,5 @@ warpSkyfile.magicked FROM distRun +JOIN distTarget USING(target_id, stage, clean) JOIN warpRun ON stage_id = warp_id JOIN warpSkyfile using(warp_id) Index: /branches/eam_branches/20090820/ippTools/share/disttool_pendingfileset.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/disttool_pendingfileset.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/disttool_pendingfileset.sql (revision 25870) @@ -4,5 +4,5 @@ distRun.stage, stage_id, - distTarget.label, + distTarget.dist_group, distTarget.filter, CONCAT_WS('.', distRun.outroot, CONVERT(distRun.dist_id, CHAR)) as dist_dir, Index: /branches/eam_branches/20090820/ippTools/share/faketool_completely_processed_exp.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/faketool_completely_processed_exp.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/faketool_completely_processed_exp.sql (revision 25870) @@ -6,4 +6,6 @@ workdir, label, + data_group, + dist_group, reduction, expgroup, @@ -11,5 +13,6 @@ tess_id, end_stage, - epoch + epoch, + note FROM (SELECT Index: /branches/eam_branches/20090820/ippTools/share/faketool_queue_cam_id.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/faketool_queue_cam_id.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/faketool_queue_cam_id.sql (revision 25870) @@ -6,4 +6,6 @@ '%s', -- workdir '%s', -- label + '%s', -- data_group + '%s', -- dist_group '%s', -- reduction '%s', -- expgroup @@ -11,5 +13,6 @@ '%s', -- tess_id '%s', -- end_stage - NULL -- epoch + NULL, -- epoch + '%s' -- note FROM camRun WHERE Index: /branches/eam_branches/20090820/ippTools/share/flatcorr_camerarundone.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/flatcorr_camerarundone.sql (revision 25870) +++ /branches/eam_branches/20090820/ippTools/share/flatcorr_camerarundone.sql (revision 25870) @@ -0,0 +1,15 @@ +SELECT + corr_id, + camRun.* +FROM flatcorrRun +JOIN flatcorrCamLink + USING(corr_id) +JOIN camRun + USING(cam_id) +LEFT JOIN addRun + ON camRun.cam_id = addRun.cam_id +WHERE + flatcorrRun.state = 'new' + AND flatcorrCamLink.include = 1 + AND camRun.state = 'full' + AND addRun.cam_id is NULL Index: /branches/eam_branches/20090820/ippTools/share/flatcorr_chiprundone.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/flatcorr_chiprundone.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/flatcorr_chiprundone.sql (revision 25870) @@ -1,5 +1,4 @@ SELECT corr_id, - cam_id, chipRun.* FROM flatcorrRun @@ -14,3 +13,3 @@ AND flatcorrChipLink.include = 1 AND chipRun.state = 'full' - AND camRun.cam_id is NULL; + AND camRun.cam_id is NULL Index: /branches/eam_branches/20090820/ippTools/share/flatcorr_pendingprocess.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/flatcorr_pendingprocess.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/flatcorr_pendingprocess.sql (revision 25870) @@ -10,29 +10,29 @@ reduction, region, - chip_count, - cam_count + cam_count, + add_count FROM (SELECT flatcorrRun.*, - flatcorrChipLink.corr_id as chip_corr_id, - count(flatcorrChipLink.chip_id) as chip_count + flatcorrCamLink.corr_id as cam_corr_id, + count(flatcorrCamLink.cam_id) as cam_count FROM flatcorrRun - JOIN flatcorrChipLink - ON flatcorrChipLink.corr_id = flatcorrRun.corr_id - WHERE flatcorrChipLink.include = 1 + JOIN flatcorrCamLink + USING (corr_id) + WHERE flatcorrCamLink.include = 1 GROUP BY - flatcorrChipLink.corr_id) AS t1 + flatcorrCamLink.corr_id) AS t1 LEFT JOIN (SELECT - flatcorrCamLink.corr_id as cam_corr_id, - count(flatcorrCamLink.cam_id) as cam_count - FROM flatcorrCamLink - JOIN camRun - ON flatcorrCamLink.cam_id = camRun.cam_id - WHERE camRun.state = 'full' - AND flatcorrCamLink.include = 1 + flatcorrAddstarLink.corr_id as add_corr_id, + count(flatcorrAddstarLink.add_id) as add_count + FROM flatcorrAddstarLink + JOIN addRun + USING (add_id) + WHERE addRun.state = 'full' + AND flatcorrAddstarLink.include = 1 GROUP BY - flatcorrCamLink.corr_id) AS t2 -ON t1.chip_corr_id = t2.cam_corr_id -WHERE chip_count = cam_count + flatcorrAddstarLink.corr_id) AS t2 +ON t1.cam_corr_id = t2.add_corr_id +WHERE cam_count = add_count AND state = 'new' Index: /branches/eam_branches/20090820/ippTools/share/magicdstool_tocleanup.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/magicdstool_tocleanup.sql (revision 25870) +++ /branches/eam_branches/20090820/ippTools/share/magicdstool_tocleanup.sql (revision 25870) @@ -0,0 +1,12 @@ +SELECT + magic_ds_id, + magicDSRun.state, + magicDSRun.outroot, + camera +FROM magicDSRun + JOIN magicRun USING(magic_id) + JOIN rawExp USING(exp_id) +WHERE magicDSRun.state = 'goto_cleaned' +-- XXX: need to add fault to magicDSRun +-- AND magicDSRun.fault = 0 + Index: /branches/eam_branches/20090820/ippTools/share/pstamptool_pendingdependent.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/pstamptool_pendingdependent.sql (revision 25870) +++ /branches/eam_branches/20090820/ippTools/share/pstamptool_pendingdependent.sql (revision 25870) @@ -0,0 +1,3 @@ +SELECT * +FROM pstampDependent +WHERE state = 'new' Index: /branches/eam_branches/20090820/ippTools/share/pstamptool_pendingjob.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/pstamptool_pendingjob.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/pstamptool_pendingjob.sql (revision 25870) @@ -1,7 +1,9 @@ SELECT pstampJob.* FROM pstampJob - JOIN pstampRequest using(req_id) + JOIN pstampRequest USING(req_id) + LEFT JOIN pstampDependent USING(dep_id) WHERE pstampRequest.state = 'run' AND pstampRequest.fault = 0 AND pstampJob.state = 'run' AND pstampJob.fault = 0 + AND (dep_id = 0 OR pstampDependent.state = 'full') Index: /branches/eam_branches/20090820/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/pxadmin_create_tables.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/pxadmin_create_tables.sql (revision 25870) @@ -1,2 +1,7 @@ +CREATE TABLE dbversion ( + schema_version VARCHAR(64), + updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE pzDataStore ( camera VARCHAR(64), @@ -261,10 +266,13 @@ workdir_state VARCHAR(64), label VARCHAR(64), + data_group VARCHAR(64), + dist_group VARCHAR(64), reduction VARCHAR(64), - expgroup VARCHAR(64), + expgroup VARCHAR(64), dvodb VARCHAR(255), tess_id VARCHAR(64), end_stage VARCHAR(64), magicked BIGINT, + note VARCHAR(255), PRIMARY KEY(chip_id), KEY(chip_id), KEY(exp_id), @@ -272,4 +280,6 @@ KEY(workdir_state), KEY(label), + KEY(data_group), + KEY(dist_group), KEY(expgroup), KEY(end_stage), @@ -367,4 +377,6 @@ workdir_state VARCHAR(64), label VARCHAR(64), + data_group VARCHAR(64), + dist_group VARCHAR(64), reduction VARCHAR(64), expgroup VARCHAR(64), @@ -373,4 +385,5 @@ end_stage VARCHAR(64), magicked BIGINT, + note VARCHAR(255), PRIMARY KEY(cam_id), KEY(cam_id), @@ -379,4 +392,6 @@ KEY(workdir_state), KEY(label), + KEY(data_group), + KEY(dist_group), KEY(expgroup), KEY(end_stage), @@ -464,7 +479,10 @@ workdir VARCHAR(255), workdir_state VARCHAR(64), + reduction VARCHAR(64), label VARCHAR(64), + data_group VARCHAR(64), dvodb VARCHAR(255), - magicked BIGINT, + note VARCHAR(255), + image_only TINYINT, PRIMARY KEY(add_id), KEY(add_id), @@ -473,4 +491,5 @@ KEY(workdir_state), KEY(label), + KEY(data_group), INDEX(add_id, cam_id), FOREIGN KEY(cam_id) REFERENCES camRun(cam_id) @@ -480,5 +499,4 @@ add_id BIGINT AUTO_INCREMENT, dtime_addstar FLOAT, - n_stars INT, path_base VARCHAR(255), fault SMALLINT NOT NULL, @@ -498,4 +516,6 @@ workdir VARCHAR(255), label VARCHAR(64), + data_group VARCHAR(64), + dist_group VARCHAR(64), reduction VARCHAR(64), expgroup VARCHAR(64), @@ -504,8 +524,11 @@ end_stage VARCHAR(64), epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + note VARCHAR(255), PRIMARY KEY(fake_id), KEY(cam_id), KEY(state), KEY(label), + KEY(data_group), + KEY(dist_group), KEY(expgroup), KEY(end_stage), @@ -834,4 +857,6 @@ workdir_state VARCHAR(64), label VARCHAR(64), + data_group VARCHAR(64), + dist_group VARCHAR(64), dvodb VARCHAR(255), tess_id VARCHAR(64), @@ -840,4 +865,5 @@ registered DATETIME, magicked BIGINT, + note VARCHAR(255), PRIMARY KEY(warp_id), KEY(warp_id), @@ -847,4 +873,6 @@ KEY(workdir_state), KEY(label), + KEY(data_group), + KEY(dist_group), KEY(end_stage), INDEX(warp_id, fake_id), @@ -910,4 +938,6 @@ workdir VARCHAR(255), label VARCHAR(64), + data_group VARCHAR(64), + dist_group VARCHAR(64), reduction VARCHAR(64), dvodb VARCHAR(255), @@ -916,4 +946,5 @@ tess_id VARCHAR(64), filter VARCHAR(64), + note VARCHAR(255), PRIMARY KEY(stack_id), KEY(stack_id), @@ -921,5 +952,7 @@ KEY(skycell_id), KEY(tess_id), - KEY(label) + KEY(label), + KEY(data_group), + KEY(dist_group) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -973,4 +1006,6 @@ workdir VARCHAR(255), label VARCHAR(64), + data_group VARCHAR(64), + dist_group VARCHAR(64), reduction VARCHAR(64), dvodb VARCHAR(255), @@ -980,9 +1015,12 @@ exposure TINYINT DEFAULT 0, magicked BIGINT, + note VARCHAR(255), PRIMARY KEY(diff_id), KEY(diff_id), KEY(state), KEY(tess_id), - KEY(label) + KEY(label), + KEY(data_group), + KEY(dist_group) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1055,7 +1093,9 @@ workdir_state VARCHAR(255), label VARCHAR(64), + data_group VARCHAR(64), dvodb VARCHAR(255), registered DATETIME, fault SMALLINT, + note VARCHAR(255), PRIMARY KEY(magic_id), KEY(magic_id), @@ -1063,4 +1103,5 @@ KEY(workdir_state), KEY(label), + KEY(data_group), KEY(fault), FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id), @@ -1118,8 +1159,11 @@ cam_id BIGINT, label VARCHAR(64), + data_group VARCHAR(64), outroot VARCHAR(255), recoveryroot VARCHAR(255), re_place TINYINT, remove TINYINT, + fault SMALLINT, + note VARCHAR(255), PRIMARY KEY(magic_ds_id), KEY(magic_ds_id), @@ -1127,4 +1171,5 @@ KEY(magic_id), KEY(label), + KEY(fault), FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id), FOREIGN KEY(inv_magic_id) REFERENCES magicRun(magic_id) @@ -1182,5 +1227,5 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; --- these two tables link the flatcorrRun to the associated chip and camera runs +-- these three tables link the flatcorrRun to the associated chip, camera, and addstar runs CREATE TABLE flatcorrChipLink ( corr_id BIGINT, @@ -1201,4 +1246,15 @@ FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id), FOREIGN KEY(cam_id) REFERENCES camRun(cam_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE flatcorrAddstarLink ( + corr_id BIGINT, + cam_id BIGINT, + add_id BIGINT, + include TINYINT, + PRIMARY KEY(corr_id, cam_id, add_id), + FOREIGN KEY (corr_id) REFERENCES flatcorrRun(corr_id), + FOREIGN KEY (cam_id) REFERENCES camRun(cam_id), + FOREIGN KEY (add_id) REFERENCES addRun(add_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1250,4 +1306,5 @@ outputBase VARCHAR(255), options BIGINT, + dep_id BIGINT, PRIMARY KEY(job_id, req_id), KEY(job_id), @@ -1255,7 +1312,18 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE pstampDependent ( + dep_id BIGINT AUTO_INCREMENT, + state VARCHAR(64), + stage VARCHAR(64), + stage_id BIGINT, + imagedb VARCHAR(64), + rlabel VARCHAR(64), + no_magic TINYINT, + PRIMARY KEY(dep_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE distTarget ( target_id BIGINT AUTO_INCREMENT, - label VARCHAR(64), + dist_group VARCHAR(64), filter VARCHAR(64), stage VARCHAR(64), @@ -1264,5 +1332,5 @@ comment VARCHAR(255), PRIMARY KEY(target_id), - CONSTRAINT UNIQUE (label, filter, stage, clean) + CONSTRAINT UNIQUE (dist_group, filter, stage, clean) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1280,4 +1348,5 @@ time_stamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP, fault SMALLINT, + note VARCHAR(255), PRIMARY KEY(dist_id), KEY(state), Index: /branches/eam_branches/20090820/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/pxadmin_drop_tables.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/pxadmin_drop_tables.sql (revision 25870) @@ -1,4 +1,5 @@ SET FOREIGN_KEY_CHECKS=0; +DROP TABLE IF EXISTS dbversion; DROP TABLE IF EXISTS pzDataStore; DROP TABLE IF EXISTS summitExp; @@ -59,4 +60,5 @@ DROP TABLE IF EXISTS flatcorrChipLink; DROP TABLE IF EXISTS flatcorrCamLink; +DROP TABLE IF EXISTS flatcorrAddstarLink; DROP TABLE IF EXISTS pstampDataStore; DROP TABLE IF EXISTS pstampProject; @@ -78,4 +80,5 @@ DROP TABLE IF EXISTS publishRun; DROP TABLE IF EXISTS publishDone; +DROP TABLE IF EXISTS pstampDependent; SET FOREIGN_KEY_CHECKS=1 Index: /branches/eam_branches/20090820/ippTools/share/pxadmin_update_version.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/pxadmin_update_version.sql (revision 25870) +++ /branches/eam_branches/20090820/ippTools/share/pxadmin_update_version.sql (revision 25870) @@ -0,0 +1,1 @@ +UPDATE dbversion SET schema_version = '%s', updated = current_timestamp() Index: /branches/eam_branches/20090820/ippTools/share/warptool_runstate.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/warptool_runstate.sql (revision 25870) +++ /branches/eam_branches/20090820/ippTools/share/warptool_runstate.sql (revision 25870) @@ -0,0 +1,16 @@ +SELECT + warp_id, + warpRun.label, + warpRun.state, + warpRun.magicked, + chip_id, + chipRun.state AS chip_state, + exp_id, + magic_ds_id, + magicDSRun.state AS magic_ds_state +FROM warpRun + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + LEFT JOIN magicDSRun ON stage = 'warp' AND stage_id = warp_id AND re_place Index: /branches/eam_branches/20090820/ippTools/share/warptool_warped.sql =================================================================== --- /branches/eam_branches/20090820/ippTools/share/warptool_warped.sql (revision 25869) +++ /branches/eam_branches/20090820/ippTools/share/warptool_warped.sql (revision 25870) @@ -3,4 +3,5 @@ warpRun.state, warpRun.workdir, + warpRun.label, warpImfile.warp_skyfile_id, rawExp.exp_id, Index: /branches/eam_branches/20090820/ippTools/src/Makefile.am =================================================================== --- /branches/eam_branches/20090820/ippTools/src/Makefile.am (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/Makefile.am (revision 25870) @@ -34,4 +34,5 @@ pxmagic.h \ pxregister.h \ + pxspace.h \ pxtag.h \ pxtools.h \ @@ -78,4 +79,5 @@ pxmagic.c \ pxregister.c \ + pxspace.c \ pxtag.c \ pxtools.c \ Index: /branches/eam_branches/20090820/ippTools/src/addtool.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/addtool.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/addtool.c (revision 25870) @@ -40,9 +40,4 @@ static bool maskedMode(pxConfig *config); static bool unblockMode(pxConfig *config); -static bool pendingcleanuprunMode(pxConfig *config); -static bool pendingcleanupexpMode(pxConfig *config); -static bool donecleanupMode(pxConfig *config); -static bool exportrunMode(pxConfig *config); -static bool importrunMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -74,9 +69,4 @@ MODECASE(ADDTOOL_MODE_MASKED, maskedMode); MODECASE(ADDTOOL_MODE_UNBLOCK, unblockMode); - MODECASE(ADDTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); - MODECASE(ADDTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpMode); - MODECASE(ADDTOOL_MODE_DONECLEANUP, donecleanupMode); - MODECASE(ADDTOOL_MODE_EXPORTRUN, exportrunMode); - MODECASE(ADDTOOL_MODE_IMPORTRUN, importrunMode); default: psAbort("invalid option (this should not happen)"); @@ -106,7 +96,8 @@ psMetadata *where = psMetadataAlloc(); - pxaddGetSearchArgs (config, where); - pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); - PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + pxcamGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); if (!psListLength(where->list) && @@ -117,8 +108,13 @@ } - PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); - PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_BOOL(image_only, config->args, "-image_only", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); // find the cam_id of all the exposures that we want to queue up. @@ -130,5 +126,5 @@ } - // use psDBGenerateWhereSQL because the SQL yields an intermediate table + // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE if (where && psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); @@ -136,5 +132,4 @@ psFree(whereClause); } - psFree(where); @@ -155,4 +150,40 @@ psFree(output); return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + // loop over our list of camRun rows to check the supplied and selected dvodb and workdir values: + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + camRunRow *row = camRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + if (!dvodb && !row->dvodb) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, cam_id %" PRId64, row->label, row->cam_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, cam_id %" PRId64, row->label, row->cam_id); + psFree(output); + return false; + } + + psFree(row); } @@ -170,12 +201,11 @@ // old values in place (i.e., passing the values through). - - // loop over our list of addRun rows + // loop over our list of camRun rows for (long i = 0; i < psArrayLength(output); i++) { psMetadata *md = output->data[i]; - addRunRow *row = addRunObjectFromMetadata(md); + camRunRow *row = camRunObjectFromMetadata(md); if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into addRun"); + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); psFree(output); return false; @@ -186,8 +216,10 @@ row->cam_id, workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, label ? label : row->label, - "RECIPE", - dvodb ? dvodb : row->dvodb - + data_group ? data_group : (label ? label : row->label), + dvodb ? dvodb : row->dvodb, + note ? note : NULL, + image_only )) { if (!psDBRollback(config->dbh)) { @@ -218,12 +250,12 @@ psMetadata *where = psMetadataAlloc(); - pxaddGetSearchArgs (config, where); PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + pxcamGetSearchArgs (config, where); // most search arguments based on camera PXOPT_COPY_STR(config->args, where, "-label", "addRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-state", "addRun.state", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); - if (!psListLength(where->list) - && !psMetadataLookupBool(NULL, config->args, "-all")) { + if (!psListLength(where->list)) { psFree(where); psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); @@ -231,4 +263,16 @@ } + psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); + + // pxUpdateRun gets parameters from config->args and runs the update query + bool result = pxUpdateRun(config, where, &query, "addRun", false); + + psFree(query); + psFree(where); + + return result; + + +#ifdef notdef PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); @@ -259,4 +303,5 @@ return true; +#endif } @@ -267,9 +312,9 @@ psMetadata *where = psMetadataAlloc(); - pxaddGetSearchArgs (config, where); PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + pxcamGetSearchArgs (config, where); pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -325,62 +370,4 @@ } -/* static bool pendingimfileMode(pxConfig *config) */ -/* { */ -/* PS_ASSERT_PTR_NON_NULL(config, false); */ - -/* psMetadata *where = psMetadataAlloc(); */ -/* pxaddGetSearchArgs (config, where); */ -/* PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); */ -/* pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); */ -/* PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); */ -/* PXOPT_COPY_S64(config->args, where, "-chip_id", "addRun.chip_id", "=="); */ -/* PXOPT_COPY_STR(config->args, where, "-class_id", "addProcessedExp.class_id", "=="); */ - -/* PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); */ - -/* psString query = pxDataGet("addtool_find_pendingimfile.sql"); */ -/* if (!query) { */ -/* psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); */ -/* return false; */ -/* } */ - -/* // use psDBGenerateWhereSQL because the SQL yields an intermediate table */ -/* if (psListLength(where->list)) { */ -/* psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); */ -/* psStringAppend(&query, " AND %s", whereClause); */ -/* psFree(whereClause); */ -/* } */ -/* psFree(where); */ - -/* if (!p_psDBRunQuery(config->dbh, query)) { */ -/* psError(PS_ERR_UNKNOWN, false, "database error"); */ -/* psFree(query); */ -/* return false; */ -/* } */ -/* psFree(query); */ - -/* psArray *output = p_psDBFetchResult(config->dbh); */ -/* if (!output) { */ -/* psError(PS_ERR_UNKNOWN, false, "database error"); */ -/* return false; */ -/* } */ -/* if (!psArrayLength(output)) { */ -/* psTrace("addtool", PS_LOG_INFO, "no rows found"); */ -/* psFree(output); */ -/* return true; */ -/* } */ - -/* // negate simple so the default is true */ -/* if (!ippdbPrintMetadatas(stdout, output, "addProcessedExp", !simple)) { */ -/* psError(PS_ERR_UNKNOWN, false, "failed to print array"); */ -/* psFree(output); */ -/* return false; */ -/* } */ - -/* psFree(output); */ - -/* return true; */ -/* } */ - static bool addprocessedexpMode(pxConfig *config) { @@ -391,11 +378,7 @@ // optional - PXOPT_LOOKUP_F32(dtime_addstar, config->args, "-dtime_addstar", false, false); - - PXOPT_LOOKUP_S32(n_stars, config->args, "-n_stars", false, false); - - PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); - - PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_F32(dtime_addstar, config->args, "-dtime_addstar", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); // generate restrictions @@ -446,7 +429,6 @@ pendingRow->add_id, dtime_addstar, - n_stars, path_base, - 0 + fault ); @@ -463,5 +445,5 @@ // since there is only one exp per 'new' set addRun.state = 'full' - if (!pxaddRunSetState(config, row->add_id, "full", magicked)) { + if (!pxaddRunSetState(config, row->add_id, "full")) { psError(PS_ERR_UNKNOWN, false, "failed to change addRun.state for add_id: %" PRId64, row->add_id); psFree(row); @@ -469,38 +451,5 @@ return false; } - - // NULL for end_stage means go as far as possible - // EAM : skip here if fault != 0 - // Also, we can run fake even if tess_id is not defined -/* if (fault || (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "add"))) { */ -/* psFree(row); */ -/* psFree(pendingRow); */ -/* if (!psDBCommit(config->dbh)) { */ -/* psError(PS_ERR_UNKNOWN, false, "database error"); */ -/* return false; */ -/* } */ -/* return true; */ -/* } */ psFree(row); - // else continue on... - -/* if (!pxfakeQueueByAddID(config, */ -/* pendingRow->add_id, */ -/* pendingRow->workdir, */ -/* pendingRow->label, */ -/* pendingRow->reduction, */ -/* pendingRow->expgroup, */ -/* pendingRow->dvodb, */ -/* pendingRow->tess_id, */ -/* pendingRow->end_stage */ -/* )) { */ -/* // rollback */ -/* if (!psDBRollback(config->dbh)) { */ -/* psError(PS_ERR_UNKNOWN, false, "database error"); */ -/* } */ -/* psError(PS_ERR_UNKNOWN, false, "failed to queue new fakeRun"); */ -/* psFree(pendingRow); */ -/* return false; */ -/* } */ psFree(pendingRow); @@ -517,15 +466,16 @@ { PS_ASSERT_PTR_NON_NULL(config, false); + + // generate restrictions + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + pxcamGetSearchArgs (config, where); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); - - // generate restrictions - psMetadata *where = psMetadataAlloc(); - pxaddGetSearchArgs (config, where); - PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); - PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); if (!psListLength(where->list) && @@ -613,9 +563,9 @@ psMetadata *where = psMetadataAlloc(); - pxaddGetSearchArgs (config, where); PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + pxcamGetSearchArgs (config, where); pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); -/* PXOPT_COPY_S16(config->args, where, "-fault", "addProcessedExp.fault", "=="); */ if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { @@ -705,11 +655,8 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); - psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); -/* PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); */ -/* PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); */ + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); if (!pxSetFaultCode(config->dbh, "addProcessedExp", where, fault)) { @@ -777,5 +724,4 @@ } - static bool unblockMode(pxConfig *config) { @@ -793,324 +739,2 @@ return true; } - -static bool pendingcleanuprunMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); - - psString query = pxDataGet("addtool_pendingcleanuprun.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("addtool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "addPendingCleanupRun", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - - psFree(output); - - return true; -} - - -static bool pendingcleanupexpMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_S64(add_id, config->args, "-add_id", false, false); - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psMetadata *where = psMetadataAlloc(); - if (add_id) { - PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); - } - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - - psString query = pxDataGet("addtool_pendingcleanupexp.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("chiptool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "addPendingCleanupExp", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - - psFree(output); - - return true; -} - - -static bool donecleanupMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - - psString query = pxDataGet("addtool_donecleanup.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("addtool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "addDoneCleanup", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - - psFree(output); - - return true; -} - -bool exportrunMode(pxConfig *config) -{ - typedef struct ExportTable { - char tableName[80]; - char sqlFilename[80]; - } ExportTable; - - int numExportTables = 2; - - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_S64(det_id, config->args, "-add_id", true, false); - PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); - - FILE *f = fopen (outfile, "w"); - if (f == NULL) { - psError(PS_ERR_UNKNOWN, false, "failed to open output file"); - return false; - } - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); - - ExportTable tables [] = { - {"addRun", "addtool_export_run.sql"}, - {"addProcessedExp", "addtool_export_processed_exp.sql"}, - }; - - for (int i=0; i < numExportTables; i++) { - psString query = pxDataGet(tables[i].sqlFilename); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psError(PS_ERR_UNKNOWN, true, "no rows found"); - psFree(output); - return false; - } - - if (clean) { - if (!strcmp(tables[i].tableName, "addRun")) { - if (!pxSetStateCleaned("addRun", "state", output)) { - psFree(output); - psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s", tables[i].tableName); - return false; - } - } - } - - // we must write the export table in non-simple (true) format - if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - psFree(output); - } - - fclose (f); - - return true; -} - -bool importrunMode(pxConfig *config) -{ - unsigned int nFail; - - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); - - psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - - fprintf (stdout, "---- input ----\n"); - psMetadataPrint (stderr, input, 1); - - psMetadataItem *item = psMetadataLookup (input, "addRun"); - psAssert (item, "entry not in input?"); - psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); - - psMetadataItem *entry = psListGet (item->data.list, 0); - assert (entry); - assert (entry->type == PS_DATA_METADATA); - addRunRow *addRun = addRunObjectFromMetadata (entry->data.md); - addRunInsertObject (config->dbh, addRun); - - // fprintf (stdout, "---- add run ----\n"); - // psMetadataPrint (stderr, entry->data.md, 1); - - item = psMetadataLookup (input, "addProcessedExp"); - psAssert (item, "entry not in input?"); - psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); - - for (int i = 0; i < item->data.list->n; i++) { - psMetadataItem *entry = psListGet (item->data.list, i); - assert (entry); - assert (entry->type == PS_DATA_METADATA); - addProcessedExpRow *addProcessedExp = addProcessedExpObjectFromMetadata (entry->data.md); - addProcessedExpInsertObject (config->dbh, addProcessedExp); - - // fprintf (stdout, "---- row %d ----\n", i); - // psMetadataPrint (stderr, entry->data.md, 1); - } - - return true; -} Index: /branches/eam_branches/20090820/ippTools/src/addtoolConfig.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/addtoolConfig.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/addtoolConfig.c (revision 25870) @@ -48,10 +48,9 @@ // -definebyquery - // XXX need to allow multiple chip_ids - // XXX need to allow multiple exp_ids psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); pxcamSetSearchArgs(definebyqueryArgs); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by chipRun reduction class", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); @@ -59,60 +58,62 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-image_only", 0, "addstar image metadata but not detections", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); // -updaterun - // XXX need to allow multiple add_ids - // XXX need to allow multiple chip_ids - // XXX need to allow multiple exp_ids psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); pxcamSetSearchArgs(updaterunArgs); - psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by camRun label", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by camRun state", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); - psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); - + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by addRun label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by addRun state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by addRun reduction class", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); // -pendingexp psMetadata *pendingexpArgs = psMetadataAlloc(); - pxcamSetSearchArgs(pendingexpArgs); psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); - psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); - psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); + pxcamSetSearchArgs(pendingexpArgs); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-reduction", 0, "search by addRun reduction class", NULL); psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - - // XXX is this used? psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class", 0, "search by class", NULL); // -addprocessedexp psMetadata *addprocessedexpArgs = psMetadataAlloc(); psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "define addtool ID (required)", 0); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN); + psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); - psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN); - psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_stars", 0, "define number of stars", 0); - - psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); - psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-magicked", 0, "set magicked", 0); // -processedexp psMetadata *processedexpArgs = psMetadataAlloc(); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); pxcamSetSearchArgs(processedexpArgs); - psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by addRun reduction class", NULL); psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-all", 0, "list everything without restriction", false); - psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); -/* psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); */ // -revertprocessedexp - // XXX need to allow multiple add_ids - // XXX need to allow multiple chip_ids - // XXX need to allow multiple exp_ids psMetadata *revertprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); pxcamSetSearchArgs(revertprocessedexpArgs); - psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by addRun reduction class", NULL); @@ -121,8 +122,8 @@ // -updateprocessedexp - // XXX allow full search options? psMetadata *updateprocessedexpArgs = psMetadataAlloc(); psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by addtool ID", 0); psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); // -block @@ -137,37 +138,4 @@ psMetadata *unblockArgs = psMetadataAlloc(); psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); - - // -pendingcleanuprun - // XXX allow full search options? - psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); - psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); - psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - - // -pendingcleanupexp - // XXX allow full search options? - psMetadata *pendingcleanupexpArgs = psMetadataAlloc(); - psMetadataAddStr(pendingcleanupexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); - psMetadataAddS64(pendingcleanupexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by addstar ID", 0); - psMetadataAddBool(pendingcleanupexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddU64(pendingcleanupexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - - // -donecleanup - psMetadata *donecleanupArgs = psMetadataAlloc(); - psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); - psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - - // -exportrun - psMetadata *exportrunArgs = psMetadataAlloc(); - psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-add_id", 0, "export this addstar ID (required)", 0); - psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); - psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean", 0, "export tables as cleaned", false); - - // -importrun - psMetadata *importrunArgs = psMetadataAlloc(); - psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); - psMetadata *argSets = psMetadataAlloc(); @@ -184,9 +152,4 @@ PXOPT_ADD_MODE("-masked", "show blocked labels", ADDTOOL_MODE_MASKED, maskedArgs); PXOPT_ADD_MODE("-unblock", "remove a label block", ADDTOOL_MODE_UNBLOCK, unblockArgs); - PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", ADDTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); - PXOPT_ADD_MODE("-pendingcleanupexp", "show exps for cleanup runs", ADDTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpArgs); - PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", ADDTOOL_MODE_DONECLEANUP, donecleanupArgs); - PXOPT_ADD_MODE("-exportrun", "export run for import on other database", ADDTOOL_MODE_EXPORTRUN, exportrunArgs); - PXOPT_ADD_MODE("-importrun", "import run from metadata file", ADDTOOL_MODE_IMPORTRUN, importrunArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: /branches/eam_branches/20090820/ippTools/src/camtool.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/camtool.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/camtool.c (revision 25870) @@ -121,4 +121,6 @@ PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); @@ -126,4 +128,5 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); // find the exp_id of all the exposures that we want to queue up. @@ -212,9 +215,12 @@ workdir ? workdir : row->workdir, label ? label : row->label, + data_group ? data_group: row->data_group, + dist_group ? dist_group: row->dist_group, reduction ? reduction : row->reduction, expgroup ? expgroup : row->expgroup, dvodb ? dvodb : row->dvodb, tess_id ? tess_id : row->tess_id, - end_stage ? end_stage : row->end_stage + end_stage ? end_stage : row->end_stage, + note )) { if (!psDBRollback(config->dbh)) { @@ -251,39 +257,21 @@ PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); - if (!psListLength(where->list) - && !psMetadataLookupBool(NULL, config->args, "-all")) { + if (!psListLength(where->list)) { psFree(where); psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); return false; } - - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - - if ((!state) && (!label)) { - psError(PXTOOLS_ERR_DATA, false, "parameters are required"); - psFree(where); - return false; - } - - if (state) { - // set camRun.state to state - if (!pxcamRunSetStateByQuery(config, where, state)) { - psFree(where); - return false; - } - } - - if (label) { - // set camRun.label to label - if (!pxcamRunSetLabelByQuery(config, where, label)) { - psFree(where); - return false; - } - } - + psString query = psStringCopy("UPDATE camRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "camRun", true); + if (!result) { + psError(PXTOOLS_ERR_DATA, false, "pxUpdateRun failed"); + } + + psFree(query); psFree(where); - return true; + return result; } @@ -642,9 +630,12 @@ pendingRow->workdir, pendingRow->label, + pendingRow->data_group, + pendingRow->dist_group, pendingRow->reduction, pendingRow->expgroup, pendingRow->dvodb, pendingRow->tess_id, - pendingRow->end_stage + pendingRow->end_stage, + NULL // note does not propagate )) { // rollback @@ -656,20 +647,4 @@ return false; } - -/* if (!pxaddQueueByCamID(config, */ -/* pendingRow->cam_id, */ -/* pendingRow->workdir, */ -/* pendingRow->label, */ -/* pendingRow->reduction, */ -/* pendingRow->dvodb */ -/* )) { */ -/* // rollback */ -/* if (!psDBRollback(config->dbh)) { */ -/* psError(PS_ERR_UNKNOWN, false, "database error"); */ -/* } */ -/* psError(PS_ERR_UNKNOWN, false, "failed to queue new addRun"); */ -/* psFree(pendingRow); */ -/* return false; */ -/* } */ psFree(pendingRow); @@ -1172,4 +1147,9 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } + psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); @@ -1252,6 +1232,13 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - fprintf (stdout, "---- input ----\n"); +#ifdef notdef + fprintf (stderr, "---- input ----\n"); psMetadataPrint (stderr, input, 1); +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } psMetadataItem *item = psMetadataLookup (input, "camRun"); Index: /branches/eam_branches/20090820/ippTools/src/camtoolConfig.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/camtoolConfig.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/camtoolConfig.c (revision 25870) @@ -62,5 +62,7 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tess ID", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); // -updaterun @@ -77,5 +79,7 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); - + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); // -pendingexp psMetadata *pendingexpArgs = psMetadataAlloc(); Index: /branches/eam_branches/20090820/ippTools/src/chiptool.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/chiptool.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/chiptool.c (revision 25870) @@ -57,4 +57,5 @@ static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); +static bool runstateMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -97,4 +98,5 @@ MODECASE(CHIPTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(CHIPTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(CHIPTOOL_MODE_RUNSTATE, runstateMode); default: psAbort("invalid option (this should not happen)"); @@ -144,4 +146,7 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); // default @@ -263,4 +268,6 @@ workdir ? workdir : raw_workdir, label ? label : raw_label, + data_group ? data_group : (label ? label : raw_label), + dist_group, reduction ? reduction : raw_reduction, // expgroup ? expgroup : raw_expgroup, @@ -269,5 +276,6 @@ dvodb ? dvodb : raw_dvodb, tess_id ? tess_id : raw_tess_id, - end_stage ? end_stage : raw_end_stage + end_stage ? end_stage : raw_end_stage, + note )) { if (!psDBRollback(config->dbh)) { @@ -301,41 +309,24 @@ PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-state", "chipRun.state", "=="); - - if (!psListLength(where->list) && - !psMetadataLookupBool(NULL, config->args, "-all")) { + PXOPT_COPY_STR(config->args, where, "-data_group", "chipRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group", "chipRun.dist_group", "=="); + + if (!psListLength(where->list)) { psFree(where); - where = NULL; psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); return false; } - - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - - if ((!state) && (!label)) { - psError(PXTOOLS_ERR_DATA, false, "parameters are required"); - psFree(where); - return false; - } - - if (state) { - // set chipRun.state to state - if (!pxchipRunSetStateByQuery(config, where, state)) { - psFree(where); - return false; - } - } - - if (label) { - // set chipRun.label to label - if (!pxchipRunSetLabelByQuery(config, where, label)) { - psFree(where); - return false; - } - } - + psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id)"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "chipRun", true); + if (!result) { + psError(PXTOOLS_ERR_DATA, false, "pxUpdateRun failed"); + } + + psFree(query); psFree(where); - return true; + return result; } @@ -605,12 +596,9 @@ PXOPT_COPY_S64(config->args, where, "-magicked", "chipProcessedImfile.magicked", "=="); - PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false); - PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); - PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); - - if (!psListLength(where->list) && - !psMetadataLookupBool(NULL, config->args, "-all")) { - psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters (or -all) are required"); + psString where2 = NULL; + pxmagicAddWhere(config, &where2, "chipProcessedImfile"); + // add cuts on ra and decl if supplied + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(PXTOOLS_ERR_DATA, false, "pxSpaceAddWhere failed"); return false; } @@ -622,8 +610,14 @@ } - if (where && psListLength(where->list)) { + if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); + } else if (psMetadataLookupBool(NULL, config->args, "-all") || (faulted || where2)) { + psStringAppend(&query, " WHERE chipRun.chip_id IS NOT NULL %s", where2 ? where2 : ""); + } else { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters (or -all) are required"); + return false; } psFree(where); @@ -635,18 +629,4 @@ // don't list faulted rows psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0"); - } - if (not_destreaked) { - if (destreaked) { - psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); - return false; - } - if (magicked) { - psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); - return false; - } - psStringAppend(&query, " AND chipProcessedImfile.magicked = 0"); - } - if (destreaked) { - psStringAppend(&query, " AND chipProcessedImfile.magicked != 0"); } @@ -739,12 +719,23 @@ PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); - PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); - - if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, fault)) { + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, NULL); + + if (!state) { + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + + if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, fault)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); return false; - } - psFree(where); - + } + psFree(where); + } + else { + if (!pxchipProcessedImfileSetStateByQuery(config,where,state)) { + psError(PS_ERR_UNKNOWN, false, "failed to set chipProcessedImfile state"); + return(false); + } + } + + return true; } @@ -1114,4 +1105,7 @@ psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // chipRun, chipProcessedImfile, rawExp + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); pxAddLabelSearchArgs (config, where, "-label", "label", "=="); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); @@ -1253,9 +1247,12 @@ chipRun->workdir, chipRun->label, + chipRun->data_group, + chipRun->dist_group, chipRun->reduction, chipRun->expgroup, chipRun->dvodb, chipRun->tess_id, - chipRun->end_stage + chipRun->end_stage, + NULL // note does not propagate )) { if (!psDBRollback(config->dbh)) { @@ -1380,4 +1377,9 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } + psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); @@ -1471,4 +1473,14 @@ psVector *identifiers = psVectorAllocEmpty(16, PS_TYPE_U64); // Identifiers inserted + psMetadataIterator *iter = psMetadataIteratorAlloc(input, PS_LIST_HEAD, NULL); // Iterator + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } + // first item is the dbversion, skip it + psMetadataItem *dbversion = psMetadataGetAndIncrement(iter); + (void) dbversion; + if (!psDBTransaction(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1476,5 +1488,4 @@ } - psMetadataIterator *iter = psMetadataIteratorAlloc(input, PS_LIST_HEAD, NULL); // Iterator psMetadataItem *item; // Item from iteration while ((item = psMetadataGetAndIncrement(iter))) { @@ -1505,2 +1516,82 @@ return true; } + +static bool runstateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); + +// PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("chiptool_runstate.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); + return false; + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipRunState", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: /branches/eam_branches/20090820/ippTools/src/chiptool.h =================================================================== --- /branches/eam_branches/20090820/ippTools/src/chiptool.h (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/chiptool.h (revision 25870) @@ -47,5 +47,6 @@ CHIPTOOL_MODE_TOSCRUBBEDIMFILE, CHIPTOOL_MODE_EXPORTRUN, - CHIPTOOL_MODE_IMPORTRUN + CHIPTOOL_MODE_IMPORTRUN, + CHIPTOOL_MODE_RUNSTATE } chiptoolMode; Index: /branches/eam_branches/20090820/ippTools/src/chiptoolConfig.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/chiptoolConfig.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/chiptoolConfig.c (revision 25870) @@ -58,7 +58,9 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessellation identifier", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define distribution group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); // -updaterun @@ -67,8 +69,12 @@ psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "search by dist_group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); - psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "set data group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "set dist group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "set note", NULL); // -pendingimfile @@ -158,12 +164,12 @@ psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL); - - psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-destreaked", 0, "search for destreaked images", false); - psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for images that have not been destreaked", false); - psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-magicked", 0, "search by magicked value", 0); + pxmagicAddArguments(processedimfileArgs); + psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-all", 0, "list everything without search terms", false); psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + pxspaceAddArguments(processedimfileArgs); // -revertprocessedimfile @@ -182,5 +188,5 @@ psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); - + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); // -block psMetadata *blockArgs = psMetadataAlloc(); @@ -229,4 +235,7 @@ psMetadataAddU64(runArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddStr(runArgs, PS_LIST_TAIL, "-state", 0, "search by state (required)", NULL); + pxchipSetSearchArgs(runArgs); + psMetadataAddS64(runArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddStr(runArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); // -advanceexp @@ -268,4 +277,15 @@ psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + // -runstate + psMetadata *runstateArgs = psMetadataAlloc(); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not necessary for result", false); + + psMetadataAddU64(runstateArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadata *argSets = psMetadataAlloc(); psMetadata *modes = psMetadataAlloc(); @@ -278,4 +298,5 @@ PXOPT_ADD_MODE("-updateprocessedimfile","change procesed imfile properties", CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE,updateprocessedimfileArgs); PXOPT_ADD_MODE("-revertprocessedimfile","undo a processed imfile", CHIPTOOL_MODE_REVERTPROCESSEDIMFILE,revertprocessedimfileArgs); + PXOPT_ADD_MODE("-runstate", "list the states of chip run", CHIPTOOL_MODE_RUNSTATE, runstateArgs); PXOPT_ADD_MODE("-advanceexp", "advance completed exposures", CHIPTOOL_MODE_ADVANCEEXP, advanceexpArgs); PXOPT_ADD_MODE("-block", "set a label block", CHIPTOOL_MODE_BLOCK, blockArgs); Index: /branches/eam_branches/20090820/ippTools/src/difftool.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/difftool.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/difftool.c (revision 25870) @@ -51,4 +51,10 @@ static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, psS64 magicked); +static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state); +static bool tocleanedskyfileMode(pxConfig *config); +static bool topurgedskyfileMode(pxConfig *config); +static bool toscrubbedskyfileMode(pxConfig *config); + + # define MODECASE(caseName, func) \ @@ -88,4 +94,8 @@ MODECASE(DIFFTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(DIFFTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(DIFFTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileMode); + MODECASE(DIFFTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileMode); + MODECASE(DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileMode); + default: psAbort("invalid option (this should not happen)"); @@ -115,10 +125,13 @@ // required options - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false); PXOPT_LOOKUP_BOOL(exposure, config->args, "-exposure", false); - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-rset_eduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); // default @@ -131,4 +144,6 @@ workdir, label, + data_group ? data_group : label, + dist_group, reduction, NULL, // dvodb @@ -137,5 +152,6 @@ bothways, exposure, - false + false, + note ); if (!run) { @@ -168,14 +184,45 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-diff_id", "stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE diffRun"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "diffRun", true); + + psFree(query); + psFree(where); + + return result; + +#ifdef notdef // required options - PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", false, false); PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - - if (state) { + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + + // Copy of my hacky work around from stacktool.c + if ((state)&&(diff_id)) { // set detRun.state to state return setdiffRunState(config, diff_id, state, false); } - return true; + if ((state)&&(label)) { + return setdiffRunStateByLabel(config, label, state); + } + + psError(PS_ERR_UNKNOWN, false, "Required options not found."); + + return false; +#endif } @@ -692,8 +739,8 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false); - PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); - PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); - + + psString where2 = NULL; + pxmagicAddWhere(config, &where2, "diffSkyfile"); + pxspaceAddWhere(config, &where2, template ? "rawTemplate" : "rawInput"); psString query = pxDataGet("difftool_skyfile.sql"); if (!query) { @@ -706,24 +753,11 @@ psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); + } else if (where2) { + psStringAppend(&query, " WHERE diffRun.diff_id is not null %s", where2); } else if (!all) { psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); + return false; } psFree(where); - - if (not_destreaked) { - if (destreaked) { - psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); - return false; - } - if (magicked) { - psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); - return false; - } - psStringAppend(&query, " AND diffSkyfile.magicked = 0"); - } - if (destreaked) { - psStringAppend(&query, " AND diffSkyfile.magicked != 0"); - } - // treat limit == 0 as "no limit" @@ -878,14 +912,47 @@ } - char *query = "UPDATE diffRun SET state = '%s', magicked = %" PRId64 " WHERE diff_id = %"PRId64; - - if (!p_psDBRunQueryF(config->dbh, query, state, magicked, diff_id)) { + if (magicked) { + char *query = "UPDATE diffRun SET state = '%s', magicked = %" PRId64 " WHERE diff_id = %"PRId64; + + if (!p_psDBRunQueryF(config->dbh, query, state, magicked, diff_id)) { psError(PS_ERR_UNKNOWN, false, "failed to change state for diff_id %"PRId64, diff_id); return false; - } - + } + } + else { + char *query = "UPDATE diffRun SET state = '%s' WHERE diff_id = %"PRId64; + + if (!p_psDBRunQueryF(config->dbh, query, state, diff_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for diff_id %"PRId64, diff_id); + return false; + } + } + return true; } + + +#ifdef notdef +static bool setdiffRunStateByLabel(pxConfig *config, const char *label, const char *state) { + PS_ASSERT_PTR_NON_NULL(state,false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid diffRun state: %s", state); + return false; + } + + char *query = "UPDATE diffRun SET state = '%s' WHERE label = '%s'"; + if (!p_psDBRunQueryF(config->dbh,query,state,label)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for label %s", label); + return(false); + } + + return true; +} +#endif // Generate a single populated run @@ -895,5 +962,8 @@ const char *tess_id, // Tessellation identifier const char *label, // label + const char *data_group, // data_group + const char *dist_group, // dist_group const char *reduction, // reduction + const char *note, // note psS64 input_warp_id, // Warp identifier for input image, PS_MAX_S64 for none psS64 input_stack_id, // Stack identifier for input image, PS_MAX_S64 for none @@ -926,4 +996,6 @@ workdir, label, + data_group ? data_group : label, + dist_group, reduction, NULL, // dvodb @@ -932,5 +1004,6 @@ false, false, - 0 // magicked + 0, // magicked + note ); @@ -1006,4 +1079,6 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); // required options PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); PXOPT_LOOKUP_S64(template_warp_id, config->args, "-template_warp_id", false, false); @@ -1012,4 +1087,5 @@ PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); if (template_stack_id && template_warp_id) { @@ -1035,5 +1111,5 @@ psArray *list = psArrayAllocEmpty(16); // List of runs, to print - if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction, + if (!populatedrun(list, workdir, skycell_id, tess_id, label, data_group ? data_group : label, dist_group, reduction, note, input_warp_id ? input_warp_id : PS_MAX_S64, input_stack_id ? input_stack_id : PS_MAX_S64, @@ -1074,8 +1150,12 @@ PXOPT_COPY_STR(config->args, stackWhere, "-stack_label", "stackRun.label", "=="); - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options - PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); // option - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); // option - PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); // required option + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); // option + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); // option + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false); @@ -1313,4 +1393,6 @@ workdir, label, + data_group ? data_group : label, + dist_group, reduction, NULL, // dvodb @@ -1319,5 +1401,6 @@ false, // bothways true, // exposure - 0 // magicked + 0, // magicked + note ); @@ -1388,5 +1471,5 @@ } - if (!diffRunPrintObjects(stdout, list, !simple)) { + if (numGood && !diffRunPrintObjects(stdout, list, !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print object"); psFree(list); @@ -1469,8 +1552,12 @@ // Settings to apply to defined run - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options - PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); // option - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); // option - PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); // required options + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); // option + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); // option + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); // option + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); // option + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); // option + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); @@ -1656,6 +1743,6 @@ } - diffRunRow *run = diffRunRowAlloc(0, "reg", workdir, label, reduction, NULL, registered, - tess_id, true, true, false); // Run to insert + diffRunRow *run = diffRunRowAlloc(0, "reg", workdir, label, data_group ? data_group : label, dist_group, reduction, NULL, registered, + tess_id, true, true, false, note); // Run to insert if (!diffRunInsertObject(config->dbh, run)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1738,5 +1825,5 @@ psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); psString query = pxDataGet("difftool_pendingcleanuprun.sql"); @@ -1760,4 +1847,5 @@ } + // fprintf(stderr,"%s",query); if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1931,4 +2019,59 @@ return true; +} + +static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state) { + PS_ASSERT_PTR_NON_NULL(config, false); + + // diff_id, skycell_id are required + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + + psString query = pxDataGet("difftool_change_skyfile_data_state.sql"); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + // Uses the unconstrained (diffRun.state [NEED NOT EQUAL] run_state) version from warptool.c + + if (!p_psDBRunQueryF(config->dbh, query, data_state, diff_id, skycell_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + psFree(query); + + query = pxDataGet("difftool_change_run_state.sql"); + if (!p_psDBRunQueryF(config->dbh, query, data_state, diff_id, data_state)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + return(true); +} + +static bool tocleanedskyfileMode(pxConfig *config) { + return change_skyfile_data_state(config, "cleaned","goto_cleaned"); +} +static bool topurgedskyfileMode(pxConfig *config) { + return change_skyfile_data_state(config, "purged", "goto_purged"); +} +static bool toscrubbedskyfileMode(pxConfig *config) { + return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed"); } @@ -1956,4 +2099,8 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); @@ -2047,7 +2194,13 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - fprintf (stdout, "---- input ----\n"); +#ifdef notdef + fprintf (stderr, "---- input ----\n"); psMetadataPrint (stderr, input, 1); - +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } psMetadataItem *item = psMetadataLookup (input, "diffRun"); psAssert (item, "entry not in input?"); Index: /branches/eam_branches/20090820/ippTools/src/difftool.h =================================================================== --- /branches/eam_branches/20090820/ippTools/src/difftool.h (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/difftool.h (revision 25870) @@ -42,5 +42,9 @@ DIFFTOOL_MODE_UPDATEDIFFSKYFILE, DIFFTOOL_MODE_EXPORTRUN, - DIFFTOOL_MODE_IMPORTRUN + DIFFTOOL_MODE_IMPORTRUN, + DIFFTOOL_MODE_TOCLEANEDSKYFILE, + DIFFTOOL_MODE_TOPURGEDSKYFILE, + DIFFTOOL_MODE_TOSCRUBBEDSKYFILE + } difftoolMode; Index: /branches/eam_branches/20090820/ippTools/src/difftoolConfig.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/difftoolConfig.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/difftoolConfig.c (revision 25870) @@ -47,17 +47,26 @@ // -definerun psMetadata *definerunArgs = psMetadataAlloc(); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID (required)", NULL); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-bothways", 0, "do the subtraction both ways?", false); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-exposure", 0, "subtraction for entire exposure?", false); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); - psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-bothways", 0, "do the subtraction both ways?", false); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-exposure", 0, "subtraction for entire exposure?", false); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); - psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff ID (required)", 0); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "define by label instead of diff ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); // -addinputskyfile @@ -134,9 +143,8 @@ psMetadataAddTime(diffskyfileArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); - psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "search by magicked value", 0); - psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-destreaked", 0, "search for destreaked images", false); - psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for images that are not destreaked", false); psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by diffRun label (LIKE comparison)", NULL); psMetadataAddS16(diffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + pxmagicAddArguments(diffskyfileArgs); + pxspaceAddArguments(diffskyfileArgs); psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); @@ -175,8 +183,11 @@ psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL); psMetadataAddF32(definewarpstackArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); - psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); - psMetadataAddTime(definewarpstackArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definewarpstackArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false); psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false); @@ -198,8 +209,11 @@ psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-template_label", 0, "search by warp label for template", NULL); psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); - psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); - psMetadataAddTime(definewarpwarpArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definewarpwarpArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false); psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false); @@ -259,4 +273,19 @@ psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + // -tocleanedskyfile + psMetadata *tocleanedskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanedskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0); + psMetadataAddStr(tocleanedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + + // -topurgedskyfile + psMetadata *topurgedskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(topurgedskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0); + psMetadataAddStr(topurgedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + + // -toscrubbedskyfile + psMetadata *toscrubbedskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0); + psMetadataAddStr(toscrubbedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); @@ -296,4 +325,8 @@ PXOPT_ADD_MODE("-importrun", "import run from metadata file", DIFFTOOL_MODE_IMPORTRUN, importrunArgs); + PXOPT_ADD_MODE("-tocleanedskyfile", "set skyfile as cleaned", DIFFTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileArgs); + PXOPT_ADD_MODE("-topurgedskyfile", "set skyfile as purged", DIFFTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileArgs); + PXOPT_ADD_MODE("-toscrubbedskyfile", "set skyfile as scrubbed", DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileArgs); + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { psError(PS_ERR_UNKNOWN, true, "option parsing failed"); Index: /branches/eam_branches/20090820/ippTools/src/disttool.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/disttool.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/disttool.c (revision 25870) @@ -32,5 +32,4 @@ static bool definebyqueryMode(pxConfig *config); -static bool definerunMode(pxConfig *config); static bool updaterunMode(pxConfig *config); static bool revertrunMode(pxConfig *config); @@ -79,5 +78,4 @@ switch (config->mode) { MODECASE(DISTTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); - MODECASE(DISTTOOL_MODE_DEFINERUN, definerunMode); MODECASE(DISTTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(DISTTOOL_MODE_REVERTRUN, revertrunMode); @@ -123,46 +121,4 @@ } -static bool definerunMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // required - PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); - PXOPT_LOOKUP_S64(stage_id, config->args, "-stage_id", true, false); - PXOPT_LOOKUP_STR(outroot, config->args, "-outroot", true, false); - PXOPT_LOOKUP_S64(target_id, config->args, "-target_id", true, false); - - // optional - PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); - PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); - PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); - - // TODO: check that stage has an expected value - // XXX: all of the following concerns will be managed properly by definebyquery - - // TODO: should we check that stage_id actually exists for stage - // in magicdstool we queue off of a magic_id so the stage_id, exp_id, and cam_id get looked up - // when the run is queued - - if (!distRunInsert(config->dbh, - 0, // dist_id - target_id, - stage, - stage_id, - 0, - set_label, - outroot, - clean, - no_magic, - "new", - NULL, // time_stamp - 0 // fault - )) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - return true; -} static bool definebyqueryMode(pxConfig *config) @@ -178,4 +134,6 @@ PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(set_note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_S64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -191,4 +149,5 @@ PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false); psString query = NULL; @@ -209,4 +168,7 @@ psStringAppend(&query, " AND (chipRun.label = '%s')", label); } + if (dist_group) { + psStringAppend(&query, " AND (chipRun.dist_group = '%s')", dist_group); + } } else if (!strcmp(stage, "chip")) { magicRunType = "chipRun"; @@ -222,4 +184,7 @@ psStringAppend(&query, " AND (chipRun.label = '%s')", label); } + if (dist_group) { + psStringAppend(&query, " AND (chipRun.dist_group = '%s')", dist_group); + } } else if (!strcmp(stage, "camera")) { magicRunType = "camRun"; // This is used below to set the magicked business @@ -235,4 +200,7 @@ psStringAppend(&query, " AND (camRun.label = '%s')", label); } + if (dist_group) { + psStringAppend(&query, " AND (camRun.dist_group = '%s')", dist_group); + } } else if (!strcmp(stage, "fake")) { magicRunType = "fakeRun"; @@ -246,4 +214,7 @@ if (label) { psStringAppend(&query, " AND (fakeRun.label = '%s')", label); + } + if (dist_group) { + psStringAppend(&query, " AND (fakeRun.dist_group = '%s')", dist_group); } // fake stage doesn't require magic @@ -262,4 +233,7 @@ psStringAppend(&query, " AND (warpRun.label = '%s')", label); } + if (dist_group) { + psStringAppend(&query, " AND (warpRun.dist_group = '%s')", dist_group); + } } else if (!strcmp(stage, "diff")) { @@ -276,4 +250,7 @@ psStringAppend(&query, " AND (diffRun.label = '%s')", label); } + if (dist_group) { + psStringAppend(&query, " AND (diffRun.dist_group = '%s')", dist_group); + } } else if (!strcmp(stage, "stack")) { @@ -288,4 +265,7 @@ if (label) { psStringAppend(&query, " AND (stackRun.label = '%s')", label); + } + if (dist_group) { + psStringAppend(&query, " AND (stackRun.dist_group = '%s')", dist_group); } // stack stage doesn't require magic @@ -371,5 +351,5 @@ psS64 magic_ds_id = psMetadataLookupS64(NULL, md, "magicked"); psS64 target_id = psMetadataLookupS64(NULL, md, "target_id"); - psString target_label = psMetadataLookupStr(NULL, md, "label"); + psString stage_label = psMetadataLookupStr(NULL, md, "label"); bool clean = psMetadataLookupBool(NULL, md, "clean"); @@ -381,5 +361,5 @@ new_label = set_label; } else { - new_label = target_label; + new_label = stage_label; } distRunRow *row = distRunRowAlloc( @@ -395,5 +375,6 @@ "new", NULL, // time_stamp - 0 // fault + 0, // fault + set_note // note does not propagate ); @@ -1239,5 +1220,5 @@ // required - PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", true, false); PXOPT_LOOKUP_STR(filter, config->args, "-filter", true, false); PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); @@ -1250,5 +1231,5 @@ distTargetRow *row = distTargetRowAlloc( 0, // target_id - label, + dist_group, filter, stage, @@ -1287,5 +1268,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "=="); PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); @@ -1323,5 +1304,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "LIKE"); PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); @@ -1518,5 +1499,5 @@ PXOPT_LOOKUP_S64(target_id, config->args, "-target_id", false, false); PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false); PXOPT_LOOKUP_STR(filter, config->args, "-filter", false, false); PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); @@ -1528,6 +1509,6 @@ error = true; } - if (!label) { - psError(PS_ERR_UNKNOWN, !error, "label is required if target_id is not supplied"); + if (!dist_group) { + psError(PS_ERR_UNKNOWN, !error, "dist_group is required if target_id is not supplied"); error = true; } @@ -1539,5 +1520,5 @@ // optional PXOPT_LOOKUP_S64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); if (state) { if (strcmp(state, "enabled") && strcmp(state, "disabled")) { @@ -1555,5 +1536,5 @@ PXOPT_COPY_STR(config->args, where, "-dest_name", "rcDestination.name", "=="); PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "LIKE"); PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); @@ -1658,5 +1639,5 @@ PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "LIKE"); PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); Index: /branches/eam_branches/20090820/ippTools/src/disttool.h =================================================================== --- /branches/eam_branches/20090820/ippTools/src/disttool.h (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/disttool.h (revision 25870) @@ -26,5 +26,4 @@ DISTTOOL_MODE_NONE = 0x0, DISTTOOL_MODE_DEFINEBYQUERY, - DISTTOOL_MODE_DEFINERUN, DISTTOOL_MODE_UPDATERUN, DISTTOOL_MODE_REVERTRUN, Index: /branches/eam_branches/20090820/ippTools/src/disttoolConfig.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/disttoolConfig.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/disttoolConfig.c (revision 25870) @@ -50,4 +50,5 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed", false); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label for run", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "select by dist_group", NULL); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-dry_run", 0, "don't queue runs just display what would be selected", false); @@ -58,18 +59,9 @@ psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id", 0, "define chip_id", 0); psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define chip_id", 0); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "select by label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "select by run label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dist_group", 0, "select by dist_group", NULL); psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - - // -definerun - psMetadata *definerunArgs = psMetadataAlloc(); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-stage", 0, "define stage for bundle (required)", NULL); - psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id (required)", 0); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-outroot", 0, "define output destination (required)", NULL); - psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id (required)", 0); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-clean", 0, "build clean distribution bundle", false); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed", false); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_label", 0, "define label for run", NULL); // -updaterun @@ -139,4 +131,5 @@ psMetadataAddS64(pendingfilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit results to label", NULL); + psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "limit results to dist_group", NULL); psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-stage", 0, "limit results to runs for stage", NULL); psMetadataAddU64(pendingfilesetArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -219,5 +212,5 @@ // -definetarget psMetadata *definetargetArgs = psMetadataAlloc(); - psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-label", 0, "define label (required)", NULL); + psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-dist_group", 0, "define dist_group (required)", NULL); psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-filter", 0, "define filter (required)", NULL); psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-stage", 0, "define stage (required)", NULL); @@ -230,12 +223,13 @@ psMetadataAddS64(updatetargetArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); - psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); - psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-dist_group", 0, "define dist_group", NULL); psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-filter", 0, "define filter", NULL); + + psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); // -listtarget psMetadata *listtargetArgs = psMetadataAlloc(); psMetadataAddS64(listtargetArgs, PS_LIST_TAIL, "-target_id", 0, "list target with target_id", 0); - psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-label", 0, "list targets for label", NULL); + psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-dist_group", 0, "list targets for dist_group", NULL); psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-filter", 0, "define filter", NULL); psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-stage", 0, "list targets for stage", NULL); @@ -252,5 +246,5 @@ psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); - psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-dist_group", 0, "define dist_group", NULL); psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-filter", 0, "define filter (LIKE comparison)", NULL); psMetadataAddBool(defineinterestArgs, PS_LIST_TAIL,"-clean", 0, "list clean targets", false); @@ -271,5 +265,5 @@ psMetadataAddS64(listinterestsArgs, PS_LIST_TAIL, "-int_id", 0, "list interests with int_id", 0); psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-dest_name", 0, "list interests for destinationn name)", NULL); - psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-label", 0, "list interests for label (LIKE comparison)", NULL); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-dist_group", 0, "list interests for dist_group (LIKE comparison)", NULL); psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-filter", 0, "list interests by filter (LIKE comparison)", NULL); psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-stage", 0, "list interests for stage", NULL); @@ -284,5 +278,4 @@ PXOPT_ADD_MODE("-definebyquery", "", DISTTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); - PXOPT_ADD_MODE("-definerun", "", DISTTOOL_MODE_DEFINERUN, definerunArgs); PXOPT_ADD_MODE("-updaterun", "", DISTTOOL_MODE_UPDATERUN, updaterunArgs); PXOPT_ADD_MODE("-revertrun", "", DISTTOOL_MODE_REVERTRUN, revertrunArgs); Index: /branches/eam_branches/20090820/ippTools/src/faketool.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/faketool.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/faketool.c (revision 25870) @@ -177,4 +177,7 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); // default @@ -265,5 +268,5 @@ // queue the exp - if (!pxfakeQueueByCamID(config, cam_id, workdir, label, reduction, expgroup, dvodb, tess_id, end_stage)) { + if (!pxfakeQueueByCamID(config, cam_id, workdir, label, data_group ? data_group : label, dist_group, reduction, expgroup, dvodb, tess_id, end_stage, note)) { if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -330,7 +333,8 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-label", "fakeRun.label", "=="); - - if (!psListLength(where->list) - && !psMetadataLookupBool(NULL, config->args, "-all")) { + PXOPT_COPY_STR(config->args, where, "-data_group", "fakeRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group", "fakeRun.dist_group", "=="); + + if (!psListLength(where->list)) { psFree(where); where = NULL; @@ -339,32 +343,16 @@ } - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - - if ((!state) && (!label)) { - psError(PXTOOLS_ERR_DATA, false, "parameters are required"); - psFree(where); - return false; - } - - if (state) { - // set fakeRun.state to state - if (!pxfakeRunSetStateByQuery(config, where, state)) { - psFree(where); - return false; - } - } - - if (label) { - // set fakeRun.label to label - if (!pxfakeRunSetLabelByQuery(config, where, label)) { - psFree(where); - return false; - } - } - + psString query = psStringCopy("UPDATE fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "fakeRun", true); + if (!result) { + psError(PXTOOLS_ERR_DATA, false, "pxUpdateRun failed"); + } + + psFree(query); psFree(where); - return true; + return result; } @@ -1162,8 +1150,11 @@ fakeRun->workdir, fakeRun->label, + fakeRun->data_group, + fakeRun->dist_group, fakeRun->dvodb, fakeRun->tess_id, fakeRun->reduction, - fakeRun->end_stage + fakeRun->end_stage, + NULL // note does not propagate )) { psError(PS_ERR_UNKNOWN, false, "failed to queue warpRun"); @@ -1279,5 +1270,8 @@ return false; } - + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); @@ -1360,7 +1354,13 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - fprintf (stdout, "---- input ----\n"); +#ifdef notdef + fprintf (stderr, "---- input ----\n"); psMetadataPrint (stderr, input, 1); - +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } psMetadataItem *item = psMetadataLookup (input, "fakeRun"); psAssert (item, "entry not in input?"); Index: /branches/eam_branches/20090820/ippTools/src/faketoolConfig.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/faketoolConfig.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/faketoolConfig.c (revision 25870) @@ -93,4 +93,7 @@ psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessellation identifier", NULL); psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddBool(queueArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); psMetadataAddBool(queueArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); @@ -139,9 +142,14 @@ psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "search by data_group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); - psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); // -pendingexp Index: /branches/eam_branches/20090820/ippTools/src/flatcorr.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/flatcorr.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/flatcorr.c (revision 25870) @@ -33,4 +33,6 @@ #include "pxtools.h" #include "pxchip.h" +#include "pxcam.h" +#include "pxadd.h" #include "flatcorr.h" @@ -38,7 +40,9 @@ static bool definerunMode(pxConfig *config); static bool addchipMode(pxConfig *config); +static bool dropchipMode(pxConfig *config); static bool addcameraMode(pxConfig *config); -static bool dropchipMode(pxConfig *config); static bool dropcameraMode(pxConfig *config); +static bool advancecameraMode(pxConfig *config); +static bool advanceaddstarMode(pxConfig *config); static bool pendingprocessMode(pxConfig *config); static bool addprocessMode(pxConfig *config); @@ -46,6 +50,4 @@ static bool inputexpMode(pxConfig *config); static bool inputimfileMode(pxConfig *config); -static bool exportrunMode(pxConfig *config); -static bool importrunMode(pxConfig *config); static bool setflatcorrRunState(pxConfig *config, psS64 corr_id, const char *state); @@ -72,7 +74,9 @@ MODECASE(FLATCORR_MODE_DEFINERUN, definerunMode); MODECASE(FLATCORR_MODE_ADDCHIP, addchipMode); + MODECASE(FLATCORR_MODE_DROPCHIP, dropchipMode); MODECASE(FLATCORR_MODE_ADDCAMERA, addcameraMode); - MODECASE(FLATCORR_MODE_DROPCHIP, dropchipMode); MODECASE(FLATCORR_MODE_DROPCAMERA, dropcameraMode); + MODECASE(FLATCORR_MODE_ADVANCECAMERA, advancecameraMode); + MODECASE(FLATCORR_MODE_ADVANCEADDSTAR, advanceaddstarMode); MODECASE(FLATCORR_MODE_PENDINGPROCESS, pendingprocessMode); MODECASE(FLATCORR_MODE_ADDPROCESS, addprocessMode); @@ -80,6 +84,4 @@ MODECASE(FLATCORR_MODE_INPUTEXP, inputexpMode); MODECASE(FLATCORR_MODE_INPUTIMFILE, inputimfileMode); - MODECASE(FLATCORR_MODE_EXPORTRUN, exportrunMode); - MODECASE(FLATCORR_MODE_IMPORTRUN, importrunMode); default: psAbort("invalid option (this should not happen)"); @@ -131,4 +133,5 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(region, config->args, "-set_region", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); @@ -233,5 +236,5 @@ // queue the exp : force this to stop at the chip stage - psS64 chip_id = pxchipQueueByExpTag(config, exp_id, workdir, label, reduction, expgroup, dvodb, tess_id, "chip"); + psS64 chip_id = pxchipQueueByExpTag(config, exp_id, workdir, label, label, NULL, reduction, expgroup, dvodb, tess_id, "chip", note); if (!chip_id) { if (!psDBRollback(config->dbh)) { @@ -297,8 +300,8 @@ // start a transaction so we don't end up with an exp without any associted // imfiles - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } + // if (!psDBTransaction(config->dbh)) { + // psError(PS_ERR_UNKNOWN, false, "database error"); + // return false; + // } // create a new flatcorrRun @@ -333,4 +336,9 @@ row->corr_id = corr_id; + // if (!psDBCommit(config->dbh)) { + // psError(PS_ERR_UNKNOWN, false, "database error"); + // return false; + // } + flatcorrRunPrintObject (stdout, row, !simple); return true; @@ -378,7 +386,50 @@ } +static bool addcameraMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", true, false); + + // add a flatcorrCamLink (initial state has include = TRUE) + // XXX should add checks that the chip_id and corr_id are in ChipLink + if (!flatcorrCamLinkInsert(config->dbh, corr_id, chip_id, cam_id, 1)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool dropcameraMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", true, false); + + // UPDATE flatcorrCamLink set include = 0 where corr_id = %lld AND cam_id = %lld + psString query = pxDataGet("flatcorr_dropcamera.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, query, (long long) corr_id, (long long) cam_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + return true; +} + // select the flatcorr chip runs that have completed and for which there is no camera entry // queue a new camera run for them -static bool addcameraMode(pxConfig *config) +static bool advancecameraMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -387,4 +438,7 @@ PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "flatcorrRun.label", "=="); psString query = pxDataGet("flatcorr_chiprundone.sql"); @@ -393,4 +447,11 @@ return false; } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); // treat limit == 0 as "no limit" @@ -463,9 +524,13 @@ row->workdir, row->label, + row->data_group, + row->dist_group, row->reduction, row->expgroup, row->dvodb, row->tess_id, - "camera")) { + "camera", + NULL // note does not propragate + )) { if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -502,31 +567,7 @@ } -static bool dropcameraMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // required - PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); - PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", true, false); - - // UPDATE flatcorrCamLink set include = 0 where corr_id = %lld AND cam_id = %lld - psString query = pxDataGet("flatcorr_dropcamera.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQueryF(config->dbh, query, (long long) corr_id, (long long) cam_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - - return true; -} - -// select the flatcorr chip runs that have completed and for which there is no camera entry -// queue a new camera run for them -static bool pendingprocessMode(pxConfig *config) +// Select the flatcorr camera runs that have completed and for which there is no addstar +// entry. Queue a new addstar run for them +static bool advanceaddstarMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -534,9 +575,10 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - - psString query = pxDataGet("flatcorr_pendingprocess.sql"); + pxAddLabelSearchArgs (config, where, "-label", "flatcorrRun.label", "=="); + + psString query = pxDataGet("flatcorr_camerarundone.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -575,8 +617,177 @@ } + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "flatcorr_addcamera", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // loop over our list of chipRun rows + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + bool status; + psS64 corr_id = psMetadataLookupS64(&status, md, "corr_id"); + if (!status) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for corr_id"); + psFree(output); + return false; + } + + camRunRow *row = camRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun"); + psFree(output); + return false; + } + + // queue the exp : force image_only to be false (flatcorr is meaningless with just image info) + if (!pxaddQueueByCamID( + config, + row->cam_id, + row->workdir, + row->reduction, + row->label, + row->data_group, + row->dvodb, + NULL, // note is not propagated + 0)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue cam_id: %" PRId64, row->cam_id); + psFree(row); + psFree(output); + return false; + } + + // figure out the ID of the flatcorrRun we just created + psS64 add_id = psDBLastInsertID(config->dbh); + + // add the addRun entry to the flatcorrAddstarLink table (include is TRUE) + if (!flatcorrAddstarLinkInsert(config->dbh, corr_id, row->cam_id, add_id, 1)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return false; +} + +// select the flatcorr chip runs that have completed and for which there is no camera entry +// queue a new camera run for them +static bool pendingprocessMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + psString query = pxDataGet("flatcorr_pendingprocess.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("flatcorr", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + if (!ippdbPrintMetadatas(stdout, output, "flatcorrPending", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); return false; + } + + return true; +} + +// XXX need a fault state +static bool addprocessMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + char *query = "UPDATE flatcorrRun SET state = 'full', hostname = '%s', fault = '%hd' WHERE corr_id = %" PRId64; + + if (!p_psDBRunQueryF(config->dbh, query, hostname, code, corr_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to change state for corr_id %" PRId64, corr_id); + return false; + } + + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + + if (!setflatcorrRunState(config, corr_id, state)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set run state"); + return false; } @@ -688,41 +899,4 @@ } -// XXX need a fault state -static bool addprocessMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); - PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); - PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); - - char *query = "UPDATE flatcorrRun SET state = 'full', hostname = '%s', fault = '%hd' WHERE corr_id = %" PRId64; - - if (!p_psDBRunQueryF(config->dbh, query, hostname, code, corr_id)) { - psError(PS_ERR_UNKNOWN, false, "failed to change state for corr_id %" PRId64, corr_id); - return false; - } - - return true; -} - -static bool updaterunMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - - if (!setflatcorrRunState(config, corr_id, state)) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "failed to set run state"); - return false; - } - - return true; -} - static bool setflatcorrRunState(pxConfig *config, psS64 corr_id, const char *state) { @@ -748,144 +922,2 @@ return true; } - -bool exportrunMode(pxConfig *config) -{ - typedef struct ExportTable { - char tableName[80]; - char sqlFilename[80]; - } ExportTable; - - int numExportTables = 3; - - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_S64(det_id, config->args, "-corr_id", true, false); - PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - - FILE *f = fopen (outfile, "w"); - if (f == NULL) { - psError(PS_ERR_UNKNOWN, false, "failed to open output file"); - return false; - } - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-corr_id", "corr_id", "=="); - - ExportTable tables [] = { - {"flatcorrRun", "flatcorr_export_run.sql"}, - {"flatcorrCamLink", "flatcorr_export_cam_link.sql"}, - {"flatcorrChipLink", "flatcorr_export_chip_link.sql"}, - }; - - for (int i=0; i < numExportTables; i++) { - psString query = pxDataGet(tables[i].sqlFilename); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("regtool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - // we must write the export table in non-simple (true) format - if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - psFree(output); - } - - fclose (f); - - return true; -} - -bool importrunMode(pxConfig *config) -{ - unsigned int nFail; - psMetadataItem *item, *entry; - - int numImportTables = 3; - - char tables[3] [80] = {"flatcorrRun", "flatcorrCamLink", "flatcorrChipLink"}; - - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); - - psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - - fprintf (stdout, "---- input ----\n"); - psMetadataPrint (stderr, input, 1); - - for (int i = 0; i < numImportTables; i++) { - item = psMetadataLookup (input, tables[i]); - psAssert (item, "entry not in input?"); - psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); - - entry = psListGet (item->data.list, 0); - assert (entry); - assert (entry->type == PS_DATA_METADATA); - - switch (i) { - case 0: - { - flatcorrRunRow *flatcorrRun = flatcorrRunObjectFromMetadata (entry->data.md); - flatcorrRunInsertObject (config->dbh, flatcorrRun); - - // fprintf (stdout, "---- flatcorr run ----\n"); - // psMetadataPrint (stderr, entry->data.md, 1); - break; - } - case 1: - { - flatcorrCamLinkRow *flatcorrCamLink = flatcorrCamLinkObjectFromMetadata (entry->data.md); - flatcorrCamLinkInsertObject (config->dbh, flatcorrCamLink); - - // fprintf (stdout, "---- flatcorr cam link ----\n"); - // psMetadataPrint (stderr, entry->data.md, 1); - break; - } - case 2: - { - flatcorrChipLinkRow *flatcorrChipLink = flatcorrChipLinkObjectFromMetadata (entry->data.md); - flatcorrChipLinkInsertObject (config->dbh, flatcorrChipLink); - - // fprintf (stdout, "---- flatcorr chip link ----\n"); - // psMetadataPrint (stderr, entry->data.md, 1); - break; - } - } - } - return true; -} Index: /branches/eam_branches/20090820/ippTools/src/flatcorr.h =================================================================== --- /branches/eam_branches/20090820/ippTools/src/flatcorr.h (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/flatcorr.h (revision 25870) @@ -28,7 +28,9 @@ FLATCORR_MODE_DEFINERUN, FLATCORR_MODE_ADDCHIP, + FLATCORR_MODE_DROPCHIP, FLATCORR_MODE_ADDCAMERA, - FLATCORR_MODE_DROPCHIP, FLATCORR_MODE_DROPCAMERA, + FLATCORR_MODE_ADVANCECAMERA, + FLATCORR_MODE_ADVANCEADDSTAR, FLATCORR_MODE_PENDINGPROCESS, FLATCORR_MODE_ADDPROCESS, @@ -36,6 +38,4 @@ FLATCORR_MODE_INPUTEXP, FLATCORR_MODE_INPUTIMFILE, - FLATCORR_MODE_EXPORTRUN, - FLATCORR_MODE_IMPORTRUN } flatcorrMode; Index: /branches/eam_branches/20090820/ippTools/src/flatcorrConfig.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/flatcorrConfig.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/flatcorrConfig.c (revision 25870) @@ -65,4 +65,6 @@ // -definerun psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-inst", 0, "define camera", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-det_type", 0, "define detrend type to be generated", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); @@ -88,7 +90,7 @@ // -addcamera psMetadata *addcameraArgs = psMetadataAlloc(); - psMetadataAddU64 (addcameraArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(addcameraArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddBool(addcameraArgs, PS_LIST_TAIL, "-pretend", 0, "use the simple output format", false); + psMetadataAddS64(addcameraArgs, PS_LIST_TAIL, "-corr_id", 0, "define Flat Correction ID (required)", 0); + psMetadataAddS64(addcameraArgs, PS_LIST_TAIL, "-chip_id", 0, "define Chip ID (required)", 0); + psMetadataAddS64(addcameraArgs, PS_LIST_TAIL, "-cam_id", 0, "define Camera ID (required)", 0); // -dropcamera @@ -96,4 +98,18 @@ psMetadataAddS64(dropcameraArgs, PS_LIST_TAIL, "-corr_id", 0, "define Flat Correction ID (required)", 0); psMetadataAddS64(dropcameraArgs, PS_LIST_TAIL, "-cam_id", 0, "define Camera ID (required)", 0); + + // -advancecamera + psMetadata *advancecameraArgs = psMetadataAlloc(); + psMetadataAddU64 (advancecameraArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(advancecameraArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(advancecameraArgs, PS_LIST_TAIL, "-pretend", 0, "use the simple output format", false); + psMetadataAddStr(advancecameraArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + + // -advanceaddstar + psMetadata *advanceaddstarArgs = psMetadataAlloc(); + psMetadataAddU64 (advanceaddstarArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(advanceaddstarArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(advanceaddstarArgs, PS_LIST_TAIL, "-pretend", 0, "use the simple output format", false); + psMetadataAddStr(advanceaddstarArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); // -pendingprocess @@ -142,17 +158,17 @@ psMetadata *modes = psMetadataAlloc(); - PXOPT_ADD_MODE("-definebyquery", "create a new, populated flat correction run", FLATCORR_MODE_DEFINEBYQUERY, definebyqueryArgs); - PXOPT_ADD_MODE("-definerun", "create a new, empty flat correction run", FLATCORR_MODE_DEFINERUN, definerunArgs); - PXOPT_ADD_MODE("-addchip", "add a chip to a flat correction run", FLATCORR_MODE_ADDCHIP, addchipArgs); - PXOPT_ADD_MODE("-addcamera", "migrate completed chips to camera stage analysis", FLATCORR_MODE_ADDCAMERA, addcameraArgs); - PXOPT_ADD_MODE("-dropchip", "drop a chip from a flat correction run", FLATCORR_MODE_DROPCHIP, dropchipArgs); - PXOPT_ADD_MODE("-dropcamera", "drop an exposure (camera stage analysis)", FLATCORR_MODE_DROPCAMERA, dropcameraArgs); - PXOPT_ADD_MODE("-pendingprocess", "show flat correction runs needing to be processed", FLATCORR_MODE_PENDINGPROCESS, pendingprocessArgs); - PXOPT_ADD_MODE("-addprocess", "report completed flat correction analysis", FLATCORR_MODE_ADDPROCESS, addprocessArgs); - PXOPT_ADD_MODE("-updaterun", "change a flat calibration run's state", FLATCORR_MODE_UPDATERUN, updaterunArgs); - PXOPT_ADD_MODE("-inputexp", "list exposures for a correction run", FLATCORR_MODE_INPUTEXP, inputexpArgs); - PXOPT_ADD_MODE("-inputimfile", "list imfiles for a chip run", FLATCORR_MODE_INPUTIMFILE, inputimfileArgs); - PXOPT_ADD_MODE("-exportrun", "export run for import on other database", FLATCORR_MODE_EXPORTRUN, exportrunArgs); - PXOPT_ADD_MODE("-importrun", "import run from metadata file", FLATCORR_MODE_IMPORTRUN, importrunArgs); + PXOPT_ADD_MODE("-definebyquery", "create a new, populated flat correction run", FLATCORR_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definerun", "create a new, empty flat correction run", FLATCORR_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-addchip", "add an existing chip run to a flat correction run", FLATCORR_MODE_ADDCHIP, addchipArgs); + PXOPT_ADD_MODE("-addcamera", "add an existing camera run to a flat correction run", FLATCORR_MODE_ADDCAMERA, addcameraArgs); + PXOPT_ADD_MODE("-advancecamera", "migrate completed chips to camera stage analysis", FLATCORR_MODE_ADVANCECAMERA, advancecameraArgs); + PXOPT_ADD_MODE("-advanceaddstar", "migrate completed exposures to addstar processing", FLATCORR_MODE_ADVANCEADDSTAR, advanceaddstarArgs); + PXOPT_ADD_MODE("-dropchip", "drop a chip from a flat correction run", FLATCORR_MODE_DROPCHIP, dropchipArgs); + PXOPT_ADD_MODE("-dropcamera", "drop an exposure (camera stage analysis)", FLATCORR_MODE_DROPCAMERA, dropcameraArgs); + PXOPT_ADD_MODE("-pendingprocess", "show flat correction runs needing to be processed", FLATCORR_MODE_PENDINGPROCESS, pendingprocessArgs); + PXOPT_ADD_MODE("-addprocess", "report completed flat correction analysis", FLATCORR_MODE_ADDPROCESS, addprocessArgs); + PXOPT_ADD_MODE("-updaterun", "change a flat calibration run's state", FLATCORR_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-inputexp", "list exposures for a correction run", FLATCORR_MODE_INPUTEXP, inputexpArgs); + PXOPT_ADD_MODE("-inputimfile", "list imfiles for a chip run", FLATCORR_MODE_INPUTIMFILE, inputimfileArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: /branches/eam_branches/20090820/ippTools/src/magicdstool.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/magicdstool.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/magicdstool.c (revision 25870) @@ -33,5 +33,4 @@ static bool definebyqueryMode(pxConfig *config); -static psS64 definerunMode(pxConfig *config); static bool updaterunMode(pxConfig *config); static bool todestreakMode(pxConfig *config); @@ -43,7 +42,7 @@ static bool torevertMode(pxConfig *config); static bool completedrevertMode(pxConfig *config); - -static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, const char *state); -static bool magicDSGetIDs(pxConfig *config, psString stage, psS64 magic_id, psS64 *stage_id, psS64 *cam_id); +static bool tocleanupMode(pxConfig *config); + +static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psMetadata *where, const char *state); # define MODECASE(caseName, func) \ @@ -66,5 +65,4 @@ switch (config->mode) { MODECASE(MAGICDSTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); - MODECASE(MAGICDSTOOL_MODE_DEFINERUN, definerunMode); MODECASE(MAGICDSTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(MAGICDSTOOL_MODE_TODESTREAK, todestreakMode); @@ -76,4 +74,5 @@ MODECASE(MAGICDSTOOL_MODE_TOREVERT, torevertMode); MODECASE(MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertMode); + MODECASE(MAGICDSTOOL_MODE_TOCLEANUP, tocleanupMode); default: psAbort("invalid option (this should not happen)"); @@ -108,4 +107,6 @@ PXOPT_LOOKUP_BOOL(noreplace, config->args, "-noreplace", false); PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); PXOPT_LOOKUP_BOOL(dry_run, config->args, "-dry_run", false); @@ -246,4 +247,5 @@ psS64 cam_id = psMetadataLookupS64(NULL, row, "cam_id"); psString magicRunLabel = psMetadataLookupStr(NULL, row, "label"); + psString magicRunDataGroup = psMetadataLookupStr(NULL, row, "data_group"); psString magicRunWorkdir = psMetadataLookupStr(NULL, row, "workdir"); @@ -267,8 +269,11 @@ cam_id, set_label ? set_label : magicRunLabel, + set_data_group ? set_data_group : magicRunDataGroup, outroot, recoveryroot, noreplace ? 0 :1, // re_place - 0); // remove + 0, // remove + 0, // fault + note); // remove psFree(outroot); @@ -313,81 +318,43 @@ } -static psS64 definerunMode(pxConfig *config) + +static bool updaterunMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); // required - PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); - PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); - PXOPT_LOOKUP_STR(outroot, config->args, "-outroot", true, false); - - // optional - PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false); - PXOPT_LOOKUP_BOOL(re_place, config->args, "-replace", false); - PXOPT_LOOKUP_BOOL(remove, config->args, "-remove", false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - - psS64 stage_id = 0, cam_id = 0; - - if (!magicDSGetIDs(config, stage, magic_id, &stage_id, &cam_id)) { - psError(PS_ERR_UNKNOWN, false, "failed to get ids"); - return false; - } - - magicDSRunRow *run = magicDSRunRowAlloc( - 0, // ID - magic_id, - 0, // inv_magic_id - "new", // state - stage, - stage_id, - cam_id, - label, - outroot, - recoveryroot, - re_place, - remove - ); - - if (!run) { - psError(PS_ERR_UNKNOWN, false, "failed to alloc magicRun object"); - return false; - } - if (!magicDSRunInsertObject(config->dbh, run)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(run); - return false; - } - - psS64 magic_ds_id = psDBLastInsertID(config->dbh); - run->magic_ds_id = magic_ds_id; - - if (!magicDSRunPrintObject(stdout, run, !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print object"); - psFree(run); - return false; - } - - psFree(run); - - return magic_id; -} - - -static bool updaterunMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // required - PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - - if (state) { - // set detRun.state to state - return setmagicDSRunState(config, magic_ds_id, state); - } - - return true; + PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + + PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", false, false); + if (magic_ds_id) { + + return setmagicDSRunState(config, magic_ds_id, NULL, state); + + } else if (!strcmp(state, "full")) { + psError(PS_ERR_UNKNOWN, true, "magic_ds_id is required to update run state to full"); + return false; + } + // we can transition by query as well + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + + + if (psListLength(where->list) < 2) { + psError(PS_ERR_UNKNOWN, true, "at least 2 search arguments are required"); + return false; + } + + + PXOPT_LOOKUP_BOOL(noreplace, config->args, "-noreplace", false); + if (!noreplace) { + psMetadataAddS32(where, PS_LIST_TAIL, "re_place", 0, ">", 0); + } + bool result = setmagicDSRunState(config, magic_ds_id, where, state); + psFree(where); + + return result; } @@ -670,68 +637,4 @@ } -static bool magicDSGetIDs(pxConfig *config, psString stage, psS64 magic_id, psS64 *stage_id, psS64 *cam_id) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(stage, false); - PS_ASSERT_PTR_NON_NULL(stage_id, false); - PS_ASSERT_PTR_NON_NULL(cam_id, false); - - int stageNum = ippStringToStage(stage);; - if (stageNum == IPP_STAGE_NONE) { - psError(PXTOOLS_ERR_DATA, false, "%s is not a valid value for stage", stage); - return false; - } - - psString query = pxDataGet("magicdstool_getrunids.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQueryF(config->dbh, query, magic_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("magicdstool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - if (psArrayLength(output) > 1) { - psError(PS_ERR_UNKNOWN, true, "unexpected number of rows found %ld for magic_id %" PRId64, - psArrayLength(output), magic_id); - return false; - } - psMetadata *row = output->data[0]; - - *cam_id = psMetadataLookupS64(NULL, row, "cam_id"); - switch (stageNum) { - case IPP_STAGE_RAW: - *stage_id = psMetadataLookupS64(NULL, row, "exp_id"); - break; - case IPP_STAGE_CHIP: - *stage_id = psMetadataLookupS64(NULL, row, "chip_id"); - break; - case IPP_STAGE_CAMERA: - *stage_id = *cam_id; - return true; - case IPP_STAGE_WARP: - *stage_id = psMetadataLookupS64(NULL, row, "warp_id"); - break; - case IPP_STAGE_DIFF: - *stage_id = psMetadataLookupS64(NULL, row, "diff_id"); - break; - } - - return true; -} static bool advancerunMode(pxConfig *config) @@ -796,5 +699,5 @@ // set magicDSRun.state to 'full' - if (!setmagicDSRunState(config, magic_ds_id, "full")) { + if (!setmagicDSRunState(config, magic_ds_id, NULL, "full")) { psError(PS_ERR_UNKNOWN, false, "failed to change magicDSRun.state for magic_ds_id: %" PRId64, magic_ds_id); @@ -994,18 +897,26 @@ } -static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value +static bool validDSRunState(const char *state) +{ if (!((strcmp(state, "new") == 0) || (strcmp(state, "full") == 0) || (strcmp(state, "restored") == 0) || (strcmp(state, "censored") == 0) || - (strcmp(state, "purged") == 0) || + (strcmp(state, "cleaned") == 0) || (strcmp(state, "goto_restored") == 0) || (strcmp(state, "goto_censored") == 0) || - (strcmp(state, "goto_purged") == 0)) + (strcmp(state, "goto_cleaned") == 0)) ) { + return false; + } else { + return true; + } +} + +static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + if (!validDSRunState(state)) { psError(PS_ERR_UNKNOWN, false, "invalid magicDSRun state: %s", state); @@ -1013,6 +924,17 @@ } - char *query = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, state, magic_ds_id)) { + psString query = psStringCopy("UPDATE magicDSRun SET state = '%s'"); + if (magic_ds_id) { + psStringAppend(&query, " WHERE magic_ds_id = %" PRId64, magic_ds_id); + } else if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search arugments are required"); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, query, state)) { psError(PS_ERR_UNKNOWN, false, "failed to change state for magic_id %" PRId64, magic_ds_id); @@ -1178,2 +1100,76 @@ +static bool tocleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("magicdstool_tocleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magicdstool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "tocleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: /branches/eam_branches/20090820/ippTools/src/magicdstool.h =================================================================== --- /branches/eam_branches/20090820/ippTools/src/magicdstool.h (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/magicdstool.h (revision 25870) @@ -26,5 +26,4 @@ MAGICDSTOOL_MODE_NONE = 0x0, MAGICDSTOOL_MODE_DEFINEBYQUERY, - MAGICDSTOOL_MODE_DEFINERUN, MAGICDSTOOL_MODE_UPDATERUN, MAGICDSTOOL_MODE_TODESTREAK, @@ -36,4 +35,5 @@ MAGICDSTOOL_MODE_TOREVERT, MAGICDSTOOL_MODE_COMPLETEDREVERT, + MAGICDSTOOL_MODE_TOCLEANUP, } MAGICDStoolMode; Index: /branches/eam_branches/20090820/ippTools/src/magicdstoolConfig.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/magicdstoolConfig.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/magicdstoolConfig.c (revision 25870) @@ -54,4 +54,6 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace input files with the destreaked versions", false); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by magicRun.label", NULL); @@ -69,19 +71,12 @@ psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - // -definerun - psMetadata *definerunArgs = psMetadataAlloc(); - psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magic_id (required)", 0); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-stage", 0, "define stage for run (required)", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-outroot", 0, "define output directory (required)", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace the input wit with the destreaked versions", false); -// psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-remove", 0, "use the simple output format", false); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); - psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magictool ID (required)", 0); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state (required)", NULL); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magictool ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-noreplace", 0, "only update runs with replace not set", false); // -addinputskyfile @@ -153,4 +148,12 @@ psMetadataAddU64(advancerunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + // -tocleanup + psMetadata *tocleanupArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanupArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); + psMetadataAddS64(tocleanupArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); + psMetadataAddStr(tocleanupArgs, PS_LIST_TAIL, "-stage", 0, "define output directory", NULL); + psMetadataAddStr(tocleanupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddU64(tocleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tocleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psFree(now); @@ -160,6 +163,4 @@ PXOPT_ADD_MODE("-definebyquery", "create magic de-streak runs from magic runs", MAGICDSTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); - PXOPT_ADD_MODE("-definerun", "create one magic de-streak run for one magic run", - MAGICDSTOOL_MODE_DEFINERUN, definerunArgs); PXOPT_ADD_MODE("-updaterun", "update state of magic de-streak run", MAGICDSTOOL_MODE_UPDATERUN, updaterunArgs); @@ -181,4 +182,6 @@ PXOPT_ADD_MODE("-completedrevert", "change state for runs that have finished reverting", MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertArgs); + PXOPT_ADD_MODE("-tocleanup", "destreak runs to clean up", + MAGICDSTOOL_MODE_TOCLEANUP, tocleanupArgs); Index: /branches/eam_branches/20090820/ippTools/src/magictool.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/magictool.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/magictool.c (revision 25870) @@ -117,4 +117,6 @@ // Optional PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-note", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); @@ -236,6 +238,17 @@ // create a new magicRun for this group - magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, inverse, "new", workdir, "dirty", label, - dvodb, registered, 0); + magicRunRow *run = magicRunRowAlloc(0, + exp_id, + diff_id, + inverse, + "new", // state + workdir, + "dirty", // workdir_state + label, + data_group ? data_group : label, + dvodb, + registered, + 0, // fault + note); if (!run) { psAbort("failed to alloc magicRun object"); @@ -331,7 +344,9 @@ "dirty", // workdir_state label, + NULL, // data_group dvodb, registered, - 0 + 0, // fault + NULL ); @@ -1449,22 +1464,22 @@ // Now queue any destreaked files to be re-verted - // note: on failure magicRestoreStage issues the rollback - if (!magicRestoreStage(config, "raw", whereClause, "goto_censored")) { + // note: on failure pxmagicRestoreStage issues the rollback + if (!pxmagicRestoreStage(config, "raw", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!magicRestoreStage(config, "chip", whereClause, "goto_censored")) { + if (!pxmagicRestoreStage(config, "chip", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!magicRestoreStage(config, "camera", whereClause, "goto_censored")) { + if (!pxmagicRestoreStage(config, "camera", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!magicRestoreStage(config, "warp", whereClause, "goto_censored")) { + if (!pxmagicRestoreStage(config, "warp", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!magicRestoreStage(config, "diff", whereClause, "goto_censored")) { + if (!pxmagicRestoreStage(config, "diff", whereClause, "goto_censored")) { psFree(whereClause); return false; Index: /branches/eam_branches/20090820/ippTools/src/pstamptool.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pstamptool.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/pstamptool.c (revision 25870) @@ -48,4 +48,7 @@ static bool projectMode(pxConfig *config); static bool modprojectMode(pxConfig *config); +static bool getdependentMode(pxConfig *config); +static bool pendingdependentMode(pxConfig *config); +static bool updatedependentMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -85,4 +88,7 @@ MODECASE(PSTAMPTOOL_MODE_MODPROJECT, modprojectMode); MODECASE(PSTAMPTOOL_MODE_PROJECT, projectMode); + MODECASE(PSTAMPTOOL_MODE_GETDEPENDENT, getdependentMode); + MODECASE(PSTAMPTOOL_MODE_PENDINGDEPENDENT, pendingdependentMode); + MODECASE(PSTAMPTOOL_MODE_UPDATEDEPENDENT, updatedependentMode); default: psAbort("invalid option (this should not happen)"); @@ -545,4 +551,5 @@ PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); PXOPT_LOOKUP_S64(options, config->args, "-options", false, false); + PXOPT_LOOKUP_S64(dep_id, config->args, "-dep_id", false, false); // unless the job is being inserted with stop state require outputBase @@ -570,5 +577,6 @@ exp_id, outputBase, - options + options, + dep_id )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -896,2 +904,150 @@ } +static bool getdependentMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + PXOPT_LOOKUP_S64(stage_id, config->args, "-stage_id", true, false); + PXOPT_LOOKUP_STR(imagedb, config->args, "-imagedb", true, false); + PXOPT_LOOKUP_STR(rlabel, config->args, "-rlabel", false, false); + PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + PXOPT_LOOKUP_BOOL(no_create, config->args, "-no_create", false); + + psString query = pxDataGet("pstamptool_pendingdependent.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (psArrayLength(output)) { + psMetadata *dep = output->data[0]; + psS64 dep_id = psMetadataLookupS64(NULL, dep, "dep_id"); + if (!dep_id) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + printf("%" PRId64 "\n", dep_id); + return true; + } + if (no_create) { + return true; + } + // no existing dependent that matches, insert one + + if (!pstampDependentInsert( + config->dbh, + 0, // dep_id + "new", // state + stage, + stage_id, + imagedb, + rlabel, + no_magic + )) { + psError(PS_ERR_UNKNOWN, false, "failed to insert pstampDependent"); + return false; + } + + psS64 dep_id = psDBLastInsertID(config->dbh); + + printf("%" PRId64 "\n", dep_id); + + return true; +} + +static bool pendingdependentMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("pstamptool_pendingdependent.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampDependent"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampDependent", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} +static bool updatedependentMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(dep_id, config->args, "-dep_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + + char *query ="UPDATE pstampDependent" + " SET state = '%s'" + " WHERE dep_id = %" PRId64; + + if (!p_psDBRunQueryF(config->dbh, query, state, dep_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + if (affected != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" + PRIu64 " rows were modified", affected); + return false; + } + + return true; +} Index: /branches/eam_branches/20090820/ippTools/src/pstamptool.h =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pstamptool.h (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/pstamptool.h (revision 25870) @@ -43,4 +43,7 @@ PSTAMPTOOL_MODE_MODPROJECT, PSTAMPTOOL_MODE_PROJECT, + PSTAMPTOOL_MODE_GETDEPENDENT, + PSTAMPTOOL_MODE_PENDINGDEPENDENT, + PSTAMPTOOL_MODE_UPDATEDEPENDENT, } pstamptoolMode; Index: /branches/eam_branches/20090820/ippTools/src/pstamptoolConfig.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pstamptoolConfig.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/pstamptoolConfig.c (revision 25870) @@ -112,4 +112,5 @@ psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure id", 0); psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-options", 0, "define options", 0); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-dep_id", 0, "define job dep_id", 0); psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0, "define job result", 0); @@ -143,4 +144,29 @@ psMetadataAddBool(revertjobArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted jobs", false); psMetadataAddU64(revertjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -getdependent + psMetadata *getdependentArgs = psMetadataAlloc(); + psMetadataAddS64(getdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define id for dependent run (required)", 0); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent run (required)", NULL); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-imagedb", 0, "define imagedb for depenent run (required)", NULL); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent run", NULL); + psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-no_magic", 0, "define no_magic", false); + psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-no_create", 0, "if no run exists do not create one", false); + + // -updatedependent + psMetadata *updatedependentArgs = psMetadataAlloc(); + psMetadataAddS64(updatedependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define id for dependent run (required)", 0); + psMetadataAddStr(updatedependentArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state (required)", NULL); + + // -pendingdependent + psMetadata *pendingdependentArgs = psMetadataAlloc(); + psMetadataAddS64(pendingdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define id for dependent run", 0); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent run", NULL); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-imagedb", 0, "define imagedb for depenent run", NULL); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent run", NULL); + psMetadataAddBool(pendingdependentArgs,PS_LIST_TAIL, "-no_magic", 0, "define no_magic", false); + psMetadataAddBool(pendingdependentArgs,PS_LIST_TAIL, "-no_create", 0, "if no run exists do not create one", false); + psMetadataAddU64(pendingdependentArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingdependentArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -addproject @@ -189,4 +215,8 @@ PXOPT_ADD_MODE("-moddatastore", "", PSTAMPTOOL_MODE_MODDATASTORE, moddatastoreArgs); + PXOPT_ADD_MODE("-getdependent", "", PSTAMPTOOL_MODE_GETDEPENDENT, getdependentArgs); + PXOPT_ADD_MODE("-updatedependent", "", PSTAMPTOOL_MODE_UPDATEDEPENDENT, updatedependentArgs); + PXOPT_ADD_MODE("-pendingdependent","", PSTAMPTOOL_MODE_PENDINGDEPENDENT, pendingdependentArgs); + PXOPT_ADD_MODE("-addproject", "", PSTAMPTOOL_MODE_ADDPROJECT, addprojectArgs); PXOPT_ADD_MODE("-modproject", "", PSTAMPTOOL_MODE_MODPROJECT, modprojectArgs); Index: /branches/eam_branches/20090820/ippTools/src/pxadd.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pxadd.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/pxadd.c (revision 25870) @@ -29,98 +29,5 @@ #include "pxadd.h" -bool pxaddSetSearchArgs (psMetadata *md) { - - psMetadataAddS64(md, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); - psMetadataAddS64(md, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); - -/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); */ -/* psMetadataAddStr(md, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); */ -/* psMetadataAddStr(md, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); */ -/* psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); */ -/* psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); */ -/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); */ -/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); */ -/* psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment", NULL); */ -/* psMetadataAddStr(md, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); */ -/* psMetadataAddStr(md, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min", 0, "define min RA (degrees) ", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max", 0, "define max RA (degrees) ", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min", 0, "define min DEC (degrees)", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max", 0, "define max DEC (degrees)", NAN); */ -/* psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min", 0, "define min exposure time", NAN); */ -/* psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max", 0, "define max exposure time", NAN); */ -/* psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); */ -/* psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min", 0, "define min background", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max", 0, "define max background", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min", 0, "define min background standard deviation", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max background standard deviation", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define min background mean standard deviation (across imfiles)", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max background mean standard deviation (across imfiles)", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min", 0, "define min altitude", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max", 0, "define max altitude", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-az_min", 0, "define min azimuth ", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-az_max", 0, "define max azimuth ", NAN); */ -/* psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); */ -/* psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); */ -/* psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); */ -/* psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); */ -/* psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); */ - - return true; -} - -bool pxaddGetSearchArgs (pxConfig *config, psMetadata *where) { - - PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); -/* PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); */ -/* PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); */ -/* PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); */ -/* PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); */ -/* PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); */ -/* PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); */ -/* PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); */ -/* PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); */ -/* PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); */ -/* PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); */ -/* PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); */ -/* PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); */ -/* PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); */ -/* PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); */ -/* PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); */ -/* PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); */ -/* PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); */ -/* PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); */ -/* PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); */ -/* PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); */ -/* PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); */ -/* PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); */ -/* PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); */ -/* PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); */ -/* PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); */ -/* PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); */ -/* PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); */ -/* PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); */ -/* PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); */ -/* PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); */ -/* PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); */ -/* PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); */ -/* PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); */ -/* PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); */ -/* PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); */ -/* PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); */ -/* PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); */ -/* PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); */ -/* PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); */ -/* PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); */ - - return true; -} - -bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state, psS64 magicked) +bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -134,8 +41,7 @@ } - char *query = "UPDATE addRun SET state = '%s', magicked = %" PRId64 " WHERE add_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, state, magicked, add_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for add_id %" PRId64, add_id); + char *query = "UPDATE addRun SET state = '%s' WHERE add_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, add_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to change state for add_id %" PRId64, add_id); return false; } @@ -216,11 +122,13 @@ } -// Need to think more about this to see what we want it to do. BROKEN bool pxaddQueueByCamID(pxConfig *config, psS64 cam_id, char *workdir, + char *reduction, char *label, - char *recipe, - char *dvodb) + char *data_group, + char *dvodb, + char *note, + bool image_only) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -236,15 +144,19 @@ } } - // fprintf(stderr,"%s",query); + // queue the exp - // XXX chip_id is being cast here work around psS64 have a different type - // different on 32/64 + // Note: cam_id is being cast here work around psS64 have a different type different + // on 32/64 if (!p_psDBRunQueryF(config->dbh, query, "new", // state - workdir ? workdir : "NULL", + workdir ? workdir : "NULL", "dirty", //workdir_state - label ? label : "NULL", - dvodb ? dvodb : "NULL", - (long long)cam_id + reduction? reduction : "NULL", + label ? label : "NULL", + data_group ? data_group : "NULL", + dvodb ? dvodb : "NULL", + note ? note : "NULL", + image_only, + (long long) cam_id )) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: /branches/eam_branches/20090820/ippTools/src/pxadd.h =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pxadd.h (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/pxadd.h (revision 25870) @@ -25,20 +25,18 @@ #include "pxtools.h" -bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state, psS64 magicked); +bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state); bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); bool pxaddRunSetLabel(pxConfig *config, psS64 add_id, const char *label); bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); -bool pxaddSetSearchArgs (psMetadata *md); -bool pxaddGetSearchArgs (pxConfig *config, psMetadata *where); - -// Likely BROKEN bool pxaddQueueByCamID(pxConfig *config, - psS64 cam_id, - char *workdir, - char *label, - char *recipe, - char *dvodb); - + psS64 cam_id, + char *workdir, + char *reduction, + char *label, + char *data_group, + char *dvodb, + char *note, + bool image_only); #endif // PXADD_H Index: /branches/eam_branches/20090820/ippTools/src/pxadmin.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pxadmin.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/pxadmin.c (revision 25870) @@ -2,5 +2,5 @@ * pxadmin.c * - * Copyright (C) 2006-2008 Joshua Hoblitt + * Copyright (C) 2006-2009 Joshua Hoblitt * * This program is free software; you can redistribute it and/or modify it @@ -31,4 +31,5 @@ bool createMirrorMode(pxConfig *config); bool deleteMode(pxConfig *config); +static bool insert_dbversion(pxConfig * config, const char *versionString); static bool runMultipleStatments(pxConfig *config, const char *query); @@ -113,4 +114,9 @@ psFree(query); + if (!insert_dbversion(config, PACKAGE_VERSION)) { + psError(PS_ERR_UNKNOWN, false, "failed to set database version"); + return false; + } + // COMMIT if (!psDBCommit(config->dbh)) { @@ -148,4 +154,9 @@ } psFree(query); + + if (!insert_dbversion(config, PACKAGE_VERSION)) { + psError(PS_ERR_UNKNOWN, false, "failed to set database version"); + return false; + } // COMMIT @@ -262,2 +273,35 @@ return true; } + +#ifdef notdef +static bool update_dbversion(pxConfig * config, const char *versionString) +{ + psString query = pxDataGet("pxadmin_update_version.sql"); + if (!query) { + psError(PS_ERR_UNKNOWN, false, "failed to retrieve SQL statement"); + psFree(query); + return false; + } + if (!p_psDBRunQueryF(config->dbh, query, versionString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + return true; +} +#endif +static bool insert_dbversion(pxConfig * config, const char *versionString) +{ + psString query = "INSERT INTO dbversion VALUES('%s', CURRENT_TIMESTAMP())"; + if (!query) { + psError(PS_ERR_UNKNOWN, false, "failed to retrieve SQL statement"); + psFree(query); + return false; + } + if (!p_psDBRunQueryF(config->dbh, query, versionString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + return true; +} Index: /branches/eam_branches/20090820/ippTools/src/pxcam.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pxcam.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/pxcam.c (revision 25870) @@ -141,86 +141,16 @@ } - -bool pxcamRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!pxIsValidState(state)) { - psError(PS_ERR_UNKNOWN, false, - "invalid chipRun state: %s", state); - return false; - } - - psString query = psStringCopy("UPDATE camRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET camRun.state = '%s'"); - - if (where) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, state)) { - psFree(query); - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(query); - - return true; -} - - -bool pxcamRunSetLabel(pxConfig *config, psS64 cam_id, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - char *query = "UPDATE camRun SET camRun.label = '%s' WHERE cam_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, label, cam_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for cam_id %" PRId64, cam_id); - return false; - } - - return true; -} - -bool pxcamRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - psString query = psStringCopy("UPDATE camRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET camRun.label = '%s'"); - - if (where) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, label)) { - psFree(query); - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(query); - - return true; -} - - bool pxcamQueueByChipID(pxConfig *config, psS64 chip_id, char *workdir, char *label, + char *data_group, + char *dist_group, char *recipe, char *expgroup, char *dvodb, char *tess_id, - char *end_stage) + char *end_stage, + char *note) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -245,4 +175,6 @@ "dirty", //workdir_state label ? label : "NULL", + data_group ? data_group : "NULL", + dist_group ? dist_group : "NULL", recipe ? recipe : "NULL", expgroup ? expgroup : "NULL", @@ -250,4 +182,5 @@ tess_id ? tess_id : "NULL", end_stage ? end_stage : "NULL", + note ? note : "NULL", (long long)chip_id )) { Index: /branches/eam_branches/20090820/ippTools/src/pxcam.h =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pxcam.h (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/pxcam.h (revision 25870) @@ -26,8 +26,4 @@ bool pxcamRunSetState(pxConfig *config, psS64 cam_id, const char *state, psS64 magicked); -bool pxcamRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); -bool pxcamRunSetLabel(pxConfig *config, psS64 cam_id, const char *label); -bool pxcamRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); - bool pxcamSetSearchArgs (psMetadata *md); bool pxcamGetSearchArgs (pxConfig *config, psMetadata *where); @@ -37,9 +33,12 @@ char *workdir, char *label, + char *data_group, + char *dist_group, char *recipe, char *expgroup, char *dvodb, char *tess_id, - char *end_stage); + char *end_stage, + char *note); #endif // PXCAM_H Index: /branches/eam_branches/20090820/ippTools/src/pxchip.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pxchip.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/pxchip.c (revision 25870) @@ -192,27 +192,26 @@ } - -bool pxchipRunSetLabel(pxConfig *config, psS64 chip_id, const char *label) +bool pxchipProcessedImfileSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) { PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - char *query = "UPDATE chipRun SET label = '%s' WHERE chip_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, label, chip_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for chip_id %" PRId64, chip_id); - return false; - } - - return true; -} - - -bool pxchipRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) SET label = '%s'"); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid chipProcessedImfile state: %s", state); + return false; + } + +/* if (!strcmp(state, "full")) { */ +/* // There are states that need to be met for a run to be set to full that we don't */ +/* // check here. */ +/* // for example all of the run's Imfiles must have chipProcessedImfile.data_state == "full" */ +/* // chipRun.magicked = (SUM(!chipProcessedImfile.magicked) = 0) */ +/* // so don't do allow setting the state to full */ +/* psError(PS_ERR_UNKNOWN, true, "cannot use -updaterun so set chipRun state to full"); */ +/* return false; */ +/* } */ + + psString query = psStringCopy("UPDATE chipProcessedImfile JOIN chipRun USING(chip_id) JOIN rawExp ON chipRun.exp_id = rawExp.exp_id SET state = '%s'"); if (where && psListLength(where->list) > 0) { @@ -222,5 +221,5 @@ } - if (!p_psDBRunQueryF(config->dbh, query, label)) { + if (!p_psDBRunQueryF(config->dbh, query, state)) { psFree(query); psError(PS_ERR_UNKNOWN, false, "database error"); @@ -232,5 +231,4 @@ return true; } - psS64 pxchipQueueByExpTag(pxConfig *config, @@ -238,9 +236,12 @@ const char *workdir, const char *label, + const char *data_group, + const char *dist_group, const char *reduction, const char *expgroup, const char *dvodb, const char *tess_id, - const char *end_stage) + const char *end_stage, + const char *note) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -260,4 +261,6 @@ "dirty", // workdir_state label, + data_group, + dist_group, reduction, expgroup, @@ -265,5 +268,6 @@ tess_id, end_stage, - 0 // magicked + 0, // magicked + note ) ) { Index: /branches/eam_branches/20090820/ippTools/src/pxchip.h =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pxchip.h (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/pxchip.h (revision 25870) @@ -25,8 +25,8 @@ #include "pxtools.h" + + bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const bool magicked); -bool pxchipRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); -bool pxchipRunSetLabel(pxConfig *config, psS64 chip_id, const char *label); -bool pxchipRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); +bool pxchipProcessedImfileSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); psS64 pxchipQueueByExpTag(pxConfig *config, @@ -34,9 +34,12 @@ const char *workdir, const char *label, + const char *data_group, + const char *dist_group, const char *reduction, const char *expgroup, const char *dvodb, const char *tess_id, - const char *end_stage); + const char *end_stage, + const char *note); Index: /branches/eam_branches/20090820/ippTools/src/pxfake.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pxfake.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/pxfake.c (revision 25870) @@ -51,87 +51,16 @@ } - -bool pxfakeRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!pxIsValidState(state)) { - psError(PS_ERR_UNKNOWN, false, - "invalid fakeRun state: %s", state); - return false; - } - - psString query = psStringCopy("UPDATE fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET fakeRun.state = '%s'"); - - if (where) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, state)) { - psFree(query); - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(query); - - return true; -} - - -bool pxfakeRunSetLabel(pxConfig *config, psS64 fake_id, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - char *query = "UPDATE fakeRun SET fakeRun.label = '%s' WHERE fake_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, label, fake_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for fake_id %" PRId64, fake_id); - return false; - } - - return true; -} - - -bool pxfakeRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - psString query = psStringCopy("UPDATE fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET fakeRun.label = '%s'"); - - if (where) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, label)) { - psFree(query); - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(query); - - return true; -} - - psS64 pxfakeQueueByCamID(pxConfig *config, psS64 cam_id, char *workdir, char *label, + char *data_group, + char *dist_group, char *reduction, char *expgroup, char *dvodb, char *tess_id, - char *end_stage) + char *end_stage, + char *note) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -157,4 +86,6 @@ workdir ? workdir : "NULL", label ? label : "NULL", + data_group ? data_group : "NULL", + dist_group ? dist_group : "NULL", reduction? reduction: "NULL", expgroup ? expgroup : "NULL", @@ -162,4 +93,5 @@ tess_id ? tess_id : "NULL", end_stage ? end_stage : "NULL", + note ? note : "NULL", (long long)cam_id )) { Index: /branches/eam_branches/20090820/ippTools/src/pxfake.h =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pxfake.h (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/pxfake.h (revision 25870) @@ -26,8 +26,4 @@ bool pxfakeRunSetState(pxConfig *config, psS64 fake_id, const char *state); -bool pxfakeRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); -bool pxfakeRunSetLabel(pxConfig *config, psS64 fake_id, const char *label); -bool pxfakeRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); - psS64 pxfakeQueueByCamID(pxConfig *config, @@ -35,9 +31,12 @@ char *workdir, char *label, + char *data_group, + char *dist_group, char *reduction, char *expgroup, char *dvodb, char *tess_id, - char *end_stage); + char *end_stage, + char *note); Index: /branches/eam_branches/20090820/ippTools/src/pxmagic.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pxmagic.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/pxmagic.c (revision 25870) @@ -29,5 +29,5 @@ #include "pxmagic.h" -bool magicRestoreStage(pxConfig *config, psString stage, psString whereClause, psString newState) +bool pxmagicRestoreStage(pxConfig *config, psString stage, psString whereClause, psString newState) { psString queryFile = NULL; @@ -66,2 +66,32 @@ return true; } + +bool pxmagicAddWhere(pxConfig *config, psString *out, psString table) +{ + PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); + + if (not_destreaked) { + if (destreaked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); + return false; + } + if (magicked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); + return false; + } + psStringAppend(out, " AND %s.magicked = 0", table); + } + if (destreaked) { + psStringAppend(out, " AND %s.magicked != 0", table); + } + return true; +} + +void pxmagicAddArguments(psMetadata *md) +{ + psMetadataAddBool(md, PS_LIST_TAIL, "-destreaked", 0, "search for destreaked images", false); + psMetadataAddBool(md, PS_LIST_TAIL, "-not_destreaked", 0, "search for images that have not been destreaked", false); + psMetadataAddS64(md, PS_LIST_TAIL, "-magicked", 0, "search by magicked value", 0); +} Index: /branches/eam_branches/20090820/ippTools/src/pxmagic.h =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pxmagic.h (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/pxmagic.h (revision 25870) @@ -25,5 +25,8 @@ #include "pxtools.h" -extern bool magicRestoreStage(pxConfig *config, psString stage, psString whereClause, psString newState); +extern bool pxmagicRestoreStage(pxConfig *config, psString stage, psString whereClause, psString newState); + +extern bool pxmagicAddWhere(pxConfig *config, psString *string, psString table); +extern void pxmagicAddArguments(psMetadata *md); #endif // PXMAGIC_H Index: /branches/eam_branches/20090820/ippTools/src/pxspace.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pxspace.c (revision 25870) +++ /branches/eam_branches/20090820/ippTools/src/pxspace.c (revision 25870) @@ -0,0 +1,57 @@ +/* + * pxmagic.c + * + * Copyright (C) 2009 IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +// #include +// #include +// #include + +#include "pxtools.h" + +bool pxspaceAddWhere(pxConfig *config, psString *pQuery, psString table) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(table, NULL); + + PXOPT_LOOKUP_F64(radius, config->args, "-radius", false, false); + + if (radius > 0) { + PXOPT_LOOKUP_F64(ra, config->args, "-ra", false, false); + PXOPT_LOOKUP_F64(decl, config->args, "-decl", false, false); + + ra *= PS_RAD_DEG; + decl *= PS_RAD_DEG; + radius *= PS_RAD_DEG; + + psStringAppend(pQuery, " AND (ACOS((cos(%s.decl) * cos(%lf) * cos(%s.ra - %lf)) +(sin(%s.decl) * sin(%lf))) < %lf)", + table, decl, table, ra, table, decl, radius); + } + + return true; +} + +void pxspaceAddArguments(psMetadata *md) +{ + psMetadataAddF64(md, PS_LIST_TAIL, "-radius", 0, "search for exposures within radius RA DEC (degrees)", false); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra", 0, "RA value for radius search (degrees)", false); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl", 0, "DEC value for radius search (degrees)", false); +} Index: /branches/eam_branches/20090820/ippTools/src/pxspace.h =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pxspace.h (revision 25870) +++ /branches/eam_branches/20090820/ippTools/src/pxspace.h (revision 25870) @@ -0,0 +1,26 @@ +/* + * pxmagic.h + * + * Copyright (C) 2009 IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXSPACE_H +#define PXSPACE_H 1 + +extern bool pxspaceAddWhere(pxConfig *config, psString *pQuery, psString table); +extern void pxspaceAddArguments(psMetadata *md); + +#endif // PXSPACE_H Index: /branches/eam_branches/20090820/ippTools/src/pxtools.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pxtools.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/pxtools.c (revision 25870) @@ -108,2 +108,185 @@ return true; } + +// shared code for updating the various strings for a Run +bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString table, bool has_dist_group) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(where, false); + PS_ASSERT_PTR_NON_NULL(pQuery, false); + PS_ASSERT_PTR_NON_NULL(*pQuery, false); + + // make sure that -state is not the only selection parameter + PXOPT_LOOKUP_STR(where_state, config->args, "-state", false, false); + if (where_state && (psListLength(where->list) < 2)) { + psError(PXTOOLS_ERR_DATA, true, "selection by -state alone is not allowed"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + + psString dist_group = NULL; + if (has_dist_group) { + PXOPT_LOOKUP_STR(tmp_dist_group, config->args, "-set_dist_group", false, false); + dist_group = tmp_dist_group; + } + + if ((!state) && (!label) && (!data_group) && (has_dist_group && !dist_group) && !(note)) { + psError(PXTOOLS_ERR_DATA, false, "parameters are required"); + return false; + } + + if (state && ! pxIsValidState(state)) { + psError(PXTOOLS_ERR_DATA, false, "pxIsValidState failed"); + return false; + } + + // first paramter is added with "SET param = 'value'" + // subseqent ones with ", param = 'value'" + char *separator = " SET "; + char *comma = ","; + +# define addColumn(_tab, _val) \ + do { \ + if (_val) { \ + psStringAppend(pQuery, "%s %s.%s = '%s'", separator, _tab, #_val, _val); \ + separator = comma; \ + } \ + } while (0) + + addColumn(table, state); + addColumn(table, data_group); + if (has_dist_group) { + addColumn(table, dist_group); + } + addColumn(table, note); + addColumn(table, label); + + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(pQuery, " %s", whereClause); + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, *pQuery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +bool pxLookupVersion(pxConfig *config, psArray **pArray) +{ + const char *query = "SELECT * FROM dbversion"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psFree(output); + psError(PS_ERR_UNKNOWN, true, "no rows in dbversion"); + return false; + } + if (psArrayLength(output) > 1) { + psError(PS_ERR_UNKNOWN, true, "unexpected number of rows found in dbversion: %" PRId64, + psArrayLength(output)); + return false; + } + *pArray = output; + + return true; +} + +psString pxGetDBVersion(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psArray *array = NULL; + if (!pxLookupVersion(config, &array)) { + psError(PS_ERR_UNKNOWN, false, "pxLookupVersion failed"); + return NULL; + } + psMetadata *md = array->data[0]; + if (!md) { + psError(PS_ERR_UNKNOWN, true, "output of pxLookupVersion is null"); + return NULL; + } + + psString version = psMetadataLookupStr(NULL, md, "schema_version"); + + return version; +} + +bool pxExportVersion(pxConfig *config, FILE *file) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(file, NULL); + + psArray *array = NULL; + if (!pxLookupVersion(config, &array) || !array) { + psError(PS_ERR_UNKNOWN, false, "pxLookupVersion failed"); + return false; + } + if (!ippdbPrintMetadatas(file, array, "dbversion", true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(array); + return false; + } + return true; +} + +bool pxCheckImportVersion(pxConfig *config, psMetadata *input) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(input, NULL); + + // This code was adapted from the way camtool parses the structures. + // Is this really the way to do it? + psMetadataItem *multi_item = psMetadataLookup(input, "dbversion"); + if (!multi_item || (multi_item->type != PS_DATA_METADATA_MULTI)) { + psError(PS_ERR_UNKNOWN, true, "dbversion multi not found in input"); + return false; + } + + psMetadataItem *dbversion = psListGet(multi_item->data.list, 0); + if (!dbversion) { + psError(PS_ERR_UNKNOWN, true, "dbversion not found in input"); + return false; + } + + if (!strcmp(dbversion->name, "dbversion")) { + // horray + psMetadata *md = dbversion->data.md; + psString schema_version = pxGetDBVersion(config); + if (!schema_version) { + psError(PS_ERR_UNKNOWN, false, "pxGetDBVersion failed"); + return false; + } + + psString import_version = psMetadataLookupStr(NULL, md, "schema_version"); + if (import_version && strcmp(import_version, schema_version)) { + psError(PS_ERR_UNKNOWN, true, "input file schema_version: %s does not match data base: %s", + import_version, schema_version); + return false; + } else if (!import_version) { + psError(PS_ERR_UNKNOWN, true, "input file schema_version is NULL"); + return false; + } else { + // YIPPEE this file is the same version + } + } else { + psError(PS_ERR_UNKNOWN, true, "Unexpected config dump format"); + return false; + } + + return true; +} Index: /branches/eam_branches/20090820/ippTools/src/pxtools.h =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pxtools.h (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/pxtools.h (revision 25870) @@ -47,4 +47,5 @@ #include "pxtree.h" #include "pxmagic.h" +#include "pxspace.h" # define MAX_ROWS 10e9 @@ -56,4 +57,6 @@ bool pxSetFaultCode(psDB *dbh, const char *tableName, psMetadata *where, psS16 code); +bool pxExportVersion(pxConfig *config, FILE *f); +bool pxCheckImportVersion(pxConfig *config, psMetadata *md); psExit pxerrorGetExitStatus(void); @@ -61,4 +64,6 @@ void pxUsage(FILE *stream, int argc, char **argv, const char *modeName, psMetadata *argSet); bool pxGetOptions(FILE *stream, int argc, char **argv, pxConfig *config, psMetadata *modes, psMetadata *argSets); + +bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString table, bool has_dist_group); #define PXOPT_ADD_MODE(option, comment, modeval, argset) \ Index: /branches/eam_branches/20090820/ippTools/src/pxwarp.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pxwarp.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/pxwarp.c (revision 25870) @@ -29,108 +29,15 @@ #include "pxwarp.h" -bool pxwarpRunSetState(pxConfig *config, psS64 warp_id, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!pxIsValidState(state)) { - psError(PS_ERR_UNKNOWN, false, "invalid warpRun state: %s", state); - return false; - } - - char *query = "UPDATE warpRun SET state = '%s' WHERE warp_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, state, warp_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for warp_id %" PRId64, warp_id); - return false; - } - - return true; -} - - -bool pxwarpRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!pxIsValidState(state)) { - psError(PS_ERR_UNKNOWN, false, "invalid warpRun state: %s", state); - return false; - } - - psString query = psStringCopy("UPDATE warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET warpRun.state = '%s'"); - - if (where) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - if (whereClause && strlen(whereClause) > 0) { - psStringAppend(&query, " %s", whereClause); - } - psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, state)) { - psFree(query); - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(query); - - return true; -} - - -bool pxwarpRunSetLabel(pxConfig *config, psS64 warp_id, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - char *query = "UPDATE warpRun SET warpRun.label = '%s' WHERE warp_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, label, warp_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change label for warp_id %" PRId64, warp_id); - return false; - } - - return true; -} - - -bool pxwarpRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - psString query = psStringCopy("UPDATE warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET warpRun.label = '%s'"); - - if (where) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, label)) { - psFree(query); - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(query); - - return true; -} - - bool pxwarpQueueByFakeID(pxConfig *config, psS64 fake_id, const char *workdir, const char *label, + const char *data_group, + const char *dist_group, const char *dvodb, const char *tess_id, const char *reduction, - const char *end_stage) + const char *end_stage, + const char *note) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -145,10 +52,13 @@ "dirty", // workdir_state label, + data_group, + dist_group, dvodb, tess_id, - reduction, + reduction, end_stage, NULL, // registered - 0 // magicked zero when created will get updated when warpRun goes to 'full' + 0, // magicked set to zero when created may get updated when warpRun goes to 'full' + NULL // note )) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: /branches/eam_branches/20090820/ippTools/src/pxwarp.h =================================================================== --- /branches/eam_branches/20090820/ippTools/src/pxwarp.h (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/pxwarp.h (revision 25870) @@ -25,17 +25,15 @@ #include "pxtools.h" -bool pxwarpRunSetState(pxConfig *config, psS64 warp_id, const char *state); -bool pxwarpRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); -bool pxwarpRunSetLabel(pxConfig *config, psS64 warp_id, const char *label); -bool pxwarpRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); - bool pxwarpQueueByFakeID(pxConfig *config, psS64 fake_id, const char *workdir, const char *label, + const char *data_group, + const char *dist_group, const char *dvodb, const char *tess_id, const char *reduction, - const char *end_stage); + const char *end_stage, + const char *note); #endif // PXWARP_H Index: /branches/eam_branches/20090820/ippTools/src/regtool.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/regtool.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/regtool.c (revision 25870) @@ -334,14 +334,24 @@ PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "obs_mode", "=="); PXOPT_COPY_S64(config->args, where, "-magicked", "magicked", "=="); - - PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); - PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); - PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_BOOL(ordered_by_date, config->args, "-ordered_by_date", false); + + // build where string for some parameters that don't fit PXOPT_COPY* + psString where2 = NULL; + if (!pxmagicAddWhere(config, &where2, "rawImfile")) { + psError(PXTOOLS_ERR_DATA, false, "pxSpaceAddWhere failed"); + return false; + } + if (!pxspaceAddWhere(config, &where2, "rawImfile")) { + psError(PXTOOLS_ERR_DATA, false, "pxSpaceAddWhere failed"); + return false; + } psString query = pxDataGet("regtool_processedimfile.sql"); @@ -356,6 +366,13 @@ psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); + } else if (!all && !where2) { + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; } psFree(where); + if (where2) { + psStringAppend(&query, " %s", where2); + psFree(where2); + } if (faulted) { @@ -366,20 +383,4 @@ psStringAppend(&query, " %s", "AND rawImfile.fault = 0"); } - - if (not_destreaked) { - if (destreaked) { - psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); - return false; - } - if (magicked) { - psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); - return false; - } - psStringAppend(&query, " AND rawImfile.magicked = 0"); - } - if (destreaked) { - psStringAppend(&query, " AND rawImfile.magicked != 0"); - } - // add the ORDER BY statement if desired @@ -884,9 +885,12 @@ workdir, label, + NULL, // data_group + NULL, // dist_group reduction, - NULL, // expgroup + NULL, // expgroup dvodb, tess_id, - end_stage + end_stage, + NULL // note )) { // rollback Index: /branches/eam_branches/20090820/ippTools/src/regtoolConfig.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/regtoolConfig.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/regtoolConfig.c (revision 25870) @@ -137,13 +137,15 @@ ADD_OPT(Str, processedimfileArgs, "-class_id", "search by class ID", NULL); ADD_OPT(Str, processedimfileArgs, "-filter", "search by filter", NULL); + ADD_OPT(Str, processedimfileArgs, "-obs_mode", "search by obs_mod", NULL); + ADD_OPT(Str, processedimfileArgs, "-exp_type", "search by exposure type", NULL); ADD_OPT(Time, processedimfileArgs, "-dateobs_begin", "search for exposures by time (>=)", NULL); ADD_OPT(Time, processedimfileArgs, "-dateobs_end", "search for exposures by time (<)", NULL); - ADD_OPT(S64, processedimfileArgs, "-magicked", "search by magicked value", 0); - ADD_OPT(Bool, processedimfileArgs, "-destreaked", "only return imfiles that have been destreaked", false); - ADD_OPT(Bool, processedimfileArgs, "-not_destreaked", "only return imfiles that have not been destreaked", false); ADD_OPT(U64, processedimfileArgs, "-limit", "limit result set to N items", 0); ADD_OPT(Bool, processedimfileArgs, "-faulted", "only return imfiles with a fault status set", false); + ADD_OPT(Bool, processedimfileArgs, "-all", "list without search arguments", false); ADD_OPT(Bool, processedimfileArgs, "-simple", "use the simple output format", false); ADD_OPT(Bool, processedimfileArgs, "-ordered_by_date", "force output to be in DATE order", false); + pxspaceAddArguments(processedimfileArgs); + pxmagicAddArguments(processedimfileArgs); // -revertprocessedimfile Index: /branches/eam_branches/20090820/ippTools/src/stacktool.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/stacktool.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/stacktool.c (revision 25870) @@ -109,8 +109,14 @@ // required options - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); - PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + + // optional + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); psMetadata *where = psMetadataAlloc(); @@ -160,8 +166,6 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); - PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); - - if (!psListLength(where->list) && - !psMetadataLookupBool(NULL, config->args, "-all")) { + + if (!psListLength(where->list)) { psFree(where); psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); @@ -345,4 +349,6 @@ workdir, label, + data_group ? data_group : label, + dist_group, reduction, dvodb, @@ -350,5 +356,6 @@ skycell_id, tess_id, - filter); + filter, + note); if (!stackRunInsertObject(config->dbh, run)) { @@ -468,5 +475,5 @@ // required options - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); @@ -475,10 +482,13 @@ // default PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); // options - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); - PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); // we have to support multipe exp_ids @@ -495,4 +505,6 @@ workdir, label, + data_group ? data_group : label, + dist_group, reduction, dvodb, @@ -500,5 +512,6 @@ skycell_id, tess_id, - filter); + filter, + note); if (!run) { @@ -569,13 +582,44 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false); +#ifdef notdef + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false); PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - - if (state) { + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); +#endif + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE stackRun"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "stackRun", true); + + psFree(query); + psFree(where); + + return result; + +#ifdef notdef + // Hack-y work around to make stacktool more like the other tools, without breaking other stuff (hopefully). + + if ((state)&&(stack_id)) { // set detRun.state to state return setstackRunState(config, stack_id, state); } - return true; + if ((state)&&(label)) { + return setstackRunStateByLabel(config, label, state); + } + + psError(PS_ERR_UNKNOWN, false, "Required options not found."); + return false; +#endif } @@ -1006,4 +1050,26 @@ } +#ifdef notdef +static bool setstackRunStateByLabel(pxConfig *config, const char *label, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid stackRun state: %s", state); + return false; + } + + char *query = "UPDATE stackRun SET state = '%s' WHERE label = '%s'"; + if (!p_psDBRunQueryF(config->dbh, query, state, label)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for label %s", label); + return false; + } + + return true; +} +#endif + static bool pendingcleanuprunMode(pxConfig *config) { @@ -1231,4 +1297,8 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); @@ -1317,7 +1387,13 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - fprintf (stdout, "---- input ----\n"); +#ifdef notdef + fprintf (stderr, "---- input ----\n"); psMetadataPrint (stderr, input, 1); - +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } psMetadataItem *item = psMetadataLookup (input, "stackRun"); psAssert (item, "entry not in input?"); Index: /branches/eam_branches/20090820/ippTools/src/stacktoolConfig.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/stacktoolConfig.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/stacktoolConfig.c (revision 25870) @@ -47,9 +47,12 @@ // -definebyquery psMetadata *definebyqueryArgs = psMetadataAlloc(); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); - psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define dvodb", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); @@ -84,5 +87,4 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m4_max", 0, "define max iq_m4", NAN); psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random", 0, "use this number of random elements", 0); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-min_num", 0, "minimum number of inputs", 0); psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-min_new", 0, "minimum number of new inputs", 0); @@ -93,9 +95,9 @@ // -definerun psMetadata *definerunArgs = psMetadataAlloc(); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); - psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define dvodb", NULL); + psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID (required)", NULL); @@ -106,7 +108,13 @@ // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); - psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-stack_id", 0, "define stack ID (required)", 0); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); - + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + // -addinputskyfile psMetadata *addinputskyfileArgs = psMetadataAlloc(); Index: /branches/eam_branches/20090820/ippTools/src/warptool.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/warptool.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/warptool.c (revision 25870) @@ -58,4 +58,5 @@ static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); +static bool runstateMode(pxConfig *config); static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile); @@ -108,4 +109,5 @@ MODECASE(WARPTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(WARPTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(WARPTOOL_MODE_RUNSTATE, runstateMode); default: @@ -139,4 +141,7 @@ PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-note", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); // required (no default TESS) @@ -160,4 +165,6 @@ "dirty", // workdir_state label, + data_group ? data_group : label, + dist_group, dvodb, tess_id, @@ -165,5 +172,6 @@ end_stage, registered, - 0 // magicked + 0, // magicked + note ); if (!warpRun) { @@ -251,8 +259,11 @@ PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); @@ -340,8 +351,11 @@ workdir ? workdir : row->workdir, label ? label : row->label, + data_group ? data_group: row->data_group, + dist_group ? dist_group: row->dist_group, dvodb ? dvodb : row->dvodb, tess_id ? tess_id : row->tess_id, reduction ? reduction : row->reduction, - end_stage ? end_stage : row->end_stage)) + end_stage ? end_stage : row->end_stage, + note)) { psError(PS_ERR_UNKNOWN, false, "failed to trying to queue fake_id: %" PRId64, row->fake_id); @@ -371,43 +385,23 @@ PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-state", "warpRun.state", "=="); - - if (!psListLength(where->list) - && !psMetadataLookupBool(NULL, config->args, "-all")) { + PXOPT_COPY_STR(config->args, where, "-data_group","warpRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group","warpRun.dist_group", "=="); + + if (!psListLength(where->list)) { psFree(where); - where = NULL; psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); return false; } - - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - - if ((!state) && (!label)) { - psError(PXTOOLS_ERR_DATA, false, "parameters (-state or -label) are required"); - psFree(where); - return false; - } - - if (state) { - // set warpRun.state to state - if (!pxwarpRunSetStateByQuery(config, where, state)) { - psFree(where); - return false; - } - } - - if (label) { - // set chipRun.label to label - if (!pxwarpRunSetLabelByQuery(config, where, label)) { - psFree(where); - return false; - } - } - + + psString query = psStringCopy("UPDATE warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "warpRun", true); + + psFree(query); psFree(where); - return true; -} - + return result; +} static bool expMode(pxConfig *config) @@ -1204,8 +1198,4 @@ pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); - PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false); - PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); - PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); - PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); @@ -1220,9 +1210,21 @@ } + // generate where strings for arguments that require extra processing + // beyond PXOPT_COPY* + psString where2 = NULL; + if (!pxmagicAddWhere(config, &where2, "warpSkyfile")) { + psError(PXTOOLS_ERR_DATA, false, "pxMagicAddWhere failed"); + return false; + } + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(PXTOOLS_ERR_DATA, false, "pxSpaceAddWhere failed"); + return false; + } + if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); - } else if (!all) { + } else if (!all && !where2) { psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); return false; @@ -1230,17 +1232,6 @@ psFree(where); - if (not_destreaked) { - if (destreaked) { - psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); - return false; - } - if (magicked) { - psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); - return false; - } - psStringAppend(&query, " AND warpSkyfile.magicked = 0"); - } - if (destreaked) { - psStringAppend(&query, " AND warpSkyfile.magicked != 0"); + if (where2) { + psStringAppend(&query, " %s", where2); } @@ -1712,12 +1703,30 @@ PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - - psString query = pxDataGet("warptool_updateskyfile.sql"); - - if (!p_psDBRunQueryF(config->dbh, query, fault, warp_id, skycell_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - psFree(query); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + + if (!state) { + psString query = pxDataGet("warptool_updateskyfile.sql"); + + if (!p_psDBRunQueryF(config->dbh, query, fault, warp_id, skycell_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + } + else { + if (strcmp(state,"error_cleaned") == 0) { + change_skyfile_data_state(config,"error_cleaned","goto_cleaned"); + } + else if (strcmp(state, "error_scrubbed") == 0) { + change_skyfile_data_state(config,"error_scrubbed","goto_scrubbed"); + } + else if (strcmp(state, "error_purged") == 0) { + change_skyfile_data_state(config,"error_purged","goto_purged"); + } + else { + psError(PS_ERR_UNKNOWN, false, "unhandled state given"); + return(false); + } + } return true; @@ -1744,4 +1753,8 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); @@ -1836,6 +1849,13 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - fprintf (stdout, "---- input ----\n"); +#ifdef notdef + fprintf (stderr, "---- input ----\n"); psMetadataPrint (stderr, input, 1); +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } psMetadataItem *item = psMetadataLookup (input, "warpRun"); @@ -1900,2 +1920,82 @@ return true; } + +static bool runstateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); + +// PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("warptool_runstate.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); + return false; + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpRunState", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: /branches/eam_branches/20090820/ippTools/src/warptool.h =================================================================== --- /branches/eam_branches/20090820/ippTools/src/warptool.h (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/warptool.h (revision 25870) @@ -53,5 +53,6 @@ WARPTOOL_MODE_UPDATESKYFILE, WARPTOOL_MODE_EXPORTRUN, - WARPTOOL_MODE_IMPORTRUN + WARPTOOL_MODE_IMPORTRUN, + WARPTOOL_MODE_RUNSTATE, } warptoolMode; Index: /branches/eam_branches/20090820/ippTools/src/warptoolConfig.c =================================================================== --- /branches/eam_branches/20090820/ippTools/src/warptoolConfig.c (revision 25869) +++ /branches/eam_branches/20090820/ippTools/src/warptoolConfig.c (revision 25870) @@ -100,5 +100,7 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); @@ -117,4 +119,8 @@ psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); // -updaterun @@ -128,8 +134,12 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by warpRun state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by warpRun label", NULL); - psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); - + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by warpRun data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "search by warpRun dist_group", NULL); + + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); // -exp psMetadata *expArgs = psMetadataAlloc(); @@ -218,9 +228,10 @@ psMetadataAddTime(warpedArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-filter", 0, "search for exposures by filter", NULL); - psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-destreaked", 0, "search for destreaked images", false); - psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for images that have not been destreaked", false); - psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-magicked", 0, "search by magicked value", 0); psMetadataAddS16(warpedArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + // add magic related arguments + pxmagicAddArguments(warpedArgs); + // add arguments for spatial search + pxspaceAddArguments(warpedArgs); psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); @@ -304,5 +315,6 @@ psMetadataAddStr(updateskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); psMetadataAddS16(updateskyfileArgs, PS_LIST_TAIL, "-fault", 0, "new fault code", 0); - + psMetadataAddStr(updateskyfileArgs, PS_LIST_TAIL, "-set_state", 0, "set state", 0); + // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); @@ -315,4 +327,16 @@ psMetadata *importrunArgs = psMetadataAlloc(); psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + // -runstate + psMetadata *runstateArgs = psMetadataAlloc(); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); +// psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not necessary for result", false); + + psMetadataAddU64(runstateArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -334,4 +358,5 @@ PXOPT_ADD_MODE("-advancerun", "", WARPTOOL_MODE_ADVANCERUN, advancerunArgs); PXOPT_ADD_MODE("-warped", "", WARPTOOL_MODE_WARPED, warpedArgs); + PXOPT_ADD_MODE("-runstate", "", WARPTOOL_MODE_RUNSTATE, runstateArgs); PXOPT_ADD_MODE("-revertwarped", "", WARPTOOL_MODE_REVERTWARPED, revertwarpedArgs); PXOPT_ADD_MODE("-block", "set a label block", WARPTOOL_MODE_BLOCK, blockArgs); Index: /branches/eam_branches/20090820/ippconfig/gpc1/ppImage.config =================================================================== --- /branches/eam_branches/20090820/ippconfig/gpc1/ppImage.config (revision 25869) +++ /branches/eam_branches/20090820/ippconfig/gpc1/ppImage.config (revision 25870) @@ -17,4 +17,6 @@ # use the deburned image instead of the raw, if it exists USE.DEBURNED.IMAGE BOOL TRUE # use burntool-repaired image? +USE.BEST.BURNTOOL BOOL TRUE # require the best burntooled data +MASK.BURNTOOL BOOL FALSE # Mask potential burntool trails TILTYSTREAK.APPLY BOOL FALSE # apply the 'tiltystreak' tool NOISEMAP BOOL FALSE # Apply read noise map @@ -151,4 +153,5 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails PATTERN BOOL TRUE # Subtract pattern noise FRINGE BOOL FALSE # Fringe subtraction @@ -177,4 +180,5 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails FRINGE BOOL FALSE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -204,4 +208,5 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails FRINGE BOOL FALSE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -329,4 +334,5 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails FRINGE BOOL FALSE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -949,4 +955,5 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails FRINGE BOOL FALSE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? Index: /branches/eam_branches/20090820/ippconfig/megacam/ppImage.config =================================================================== --- /branches/eam_branches/20090820/ippconfig/megacam/ppImage.config (revision 25869) +++ /branches/eam_branches/20090820/ippconfig/megacam/ppImage.config (revision 25870) @@ -21,4 +21,6 @@ # megacam needs fringe subtraction FRINGE BOOL TRUE # Fringe subtraction + +USE.DEBURNED.IMAGE BOOL FALSE # Skip burntool # this table lists extra constraints which should be applied when Index: /branches/eam_branches/20090820/ippconfig/recipes/addstar.config =================================================================== --- /branches/eam_branches/20090820/ippconfig/recipes/addstar.config (revision 25869) +++ /branches/eam_branches/20090820/ippconfig/recipes/addstar.config (revision 25870) @@ -1,12 +1,14 @@ # configurable options for addstar +# the IMAGES.ONLY option has been deprecated: set it when defining an addstar run with addtool -image-only +# it is not clear we still need an addstar recipe. -IMAGES.ONLY BOOL TRUE +# IMAGES.ONLY BOOL TRUE -FLATCORR METADATA - # IMAGES.ONLY BOOL FALSE - IMAGES.ONLY BOOL TRUE -END - -ADDSTAR METADATA - IMAGES.ONLY BOOL TRUE -END +# FLATCORR METADATA +# # IMAGES.ONLY BOOL FALSE +# IMAGES.ONLY BOOL TRUE +# END +# +# ADDSTAR METADATA +# IMAGES.ONLY BOOL TRUE +# END Index: /branches/eam_branches/20090820/ippconfig/recipes/filerules-split.mdc =================================================================== --- /branches/eam_branches/20090820/ippconfig/recipes/filerules-split.mdc (revision 25869) +++ /branches/eam_branches/20090820/ippconfig/recipes/filerules-split.mdc (revision 25870) @@ -228,5 +228,7 @@ PPSTAMP.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE PPSTAMP.OUTPUT.VARIANCE OUTPUT {OUTPUT}.wt.fits VARIANCE COMP_WT FPA TRUE NONE +# mosaic image used for making stamps of raw images. Not saved. PPSTAMP.CHIP OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF +PPSTAMP.OUTPUT.JPEG1 OUTPUT {OUTPUT}.jpg JPEG NONE FPA TRUE NONE PPSIM.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE SPLIT Index: /branches/eam_branches/20090820/ippconfig/recipes/ppImage.config =================================================================== --- /branches/eam_branches/20090820/ippconfig/recipes/ppImage.config (revision 25869) +++ /branches/eam_branches/20090820/ippconfig/recipes/ppImage.config (revision 25870) @@ -26,5 +26,5 @@ CHECK.CTE BOOL FALSE # measure CTE errors? USE.DEBURNED.IMAGE BOOL FALSE # use burntool-repaired image? -USE.BEST.BURNTOOL BOOL TRUE # require the best burntooled data +USE.BEST.BURNTOOL BOOL FALSE # require the best burntooled data TILTYSTREAK.APPLY BOOL FALSE # apply the 'tiltystreak' tool @@ -58,5 +58,5 @@ # Which regions identified by burntool to mask. 0x01 : only unfit trails, 0x02 "up" trails, 0x04 "down" trails -BURNTOOL.TRAILS U16 0x01 +BURNTOOL.TRAILS U16 0x07 # Non-linearity correction @@ -150,5 +150,4 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels - MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails FRINGE BOOL TRUE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -174,5 +173,4 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels - MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails FRINGE BOOL TRUE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -198,5 +196,4 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels - MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails FRINGE BOOL TRUE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -565,5 +562,4 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels - MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails FRINGE BOOL TRUE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -1726,5 +1722,4 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels - MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails PATTERN BOOL TRUE # Subtract pattern noise? FRINGE BOOL TRUE # Fringe subtraction Index: /branches/eam_branches/20090820/ippconfig/recipes/psphot.config =================================================================== --- /branches/eam_branches/20090820/ippconfig/recipes/psphot.config (revision 25869) +++ /branches/eam_branches/20090820/ippconfig/recipes/psphot.config (revision 25870) @@ -300,3 +300,5 @@ BACKGROUND.YBIN S32 2400 IMSTATS_NPIX S32 50000 -END + BREAK_POINT STR ENSEMBLE + PEAKS_NMAX S32 1000 # on first pass, only keep NMAX peaks (0 == all) +END Index: /branches/eam_branches/20090820/ippconfig/sdssmosaic/addstar.config =================================================================== --- /branches/eam_branches/20090820/ippconfig/sdssmosaic/addstar.config (revision 25869) +++ /branches/eam_branches/20090820/ippconfig/sdssmosaic/addstar.config (revision 25870) @@ -1,5 +1,5 @@ -IMAGES.ONLY BOOL FALSE - -ADDSTAR METADATA - IMAGES.ONLY BOOL FALSE -END +# IMAGES.ONLY BOOL FALSE +# +# ADDSTAR METADATA +# IMAGES.ONLY BOOL FALSE +# END Index: /branches/eam_branches/20090820/ippconfig/simtest/ppStack.config =================================================================== --- /branches/eam_branches/20090820/ippconfig/simtest/ppStack.config (revision 25869) +++ /branches/eam_branches/20090820/ippconfig/simtest/ppStack.config (revision 25870) @@ -1,2 +1,3 @@ PSF.MODEL STR PS_MODEL_GAUSS # Model for PSF generation DECONV.LIMIT F32 1.5 # Deconvolution fraction for rejecting entire image +IMAGE.REJ F32 0.3 # Rejected pixel fraction threshold for rejecting entire image Index: /branches/eam_branches/20090820/ppImage/src/ppImageLoop.c =================================================================== --- /branches/eam_branches/20090820/ppImage/src/ppImageLoop.c (revision 25869) +++ /branches/eam_branches/20090820/ppImage/src/ppImageLoop.c (revision 25870) @@ -51,17 +51,17 @@ } - // crosstalk measurement needs to be done on the entire chip at once, and before - // signal levels are modified by the detrending. If crosstalk measurement is - // requested, the read-level for the images is set to CHIP. - if (!ppImageMeasureCrosstalk(config, options, view)) { - ESCAPE("Unable to perform crosstalk correction"); - } - - // crosstalk correction needs to be done on the entire chip at once, and before - // signal levels are modified by the detrending. If crosstalk correction is - // requested, the read-level for the images is set to CHIP. - if (!ppImageCorrectCrosstalk(config, options, view)) { - ESCAPE("Unable to perform crosstalk correction"); - } + // crosstalk measurement needs to be done on the entire chip at once, and before + // signal levels are modified by the detrending. If crosstalk measurement is + // requested, the read-level for the images is set to CHIP. + if (!ppImageMeasureCrosstalk(config, options, view)) { + ESCAPE("Unable to perform crosstalk correction"); + } + + // crosstalk correction needs to be done on the entire chip at once, and before + // signal levels are modified by the detrending. If crosstalk correction is + // requested, the read-level for the images is set to CHIP. + if (!ppImageCorrectCrosstalk(config, options, view)) { + ESCAPE("Unable to perform crosstalk correction"); + } psTimerStart(TIMER_DETREND); @@ -259,5 +259,7 @@ psString dump_file = psMetadataLookupStr(&status, config->arguments, "DUMP_CONFIG"); if (dump_file) { - pmConfigDump(config, dump_file); + if (!pmConfigDump(config, dump_file)) { + ESCAPE("Unable to dump configuration."); + } } Index: /branches/eam_branches/20090820/ppImage/src/ppImageSetMaskBits.c =================================================================== --- /branches/eam_branches/20090820/ppImage/src/ppImageSetMaskBits.c (revision 25869) +++ /branches/eam_branches/20090820/ppImage/src/ppImageSetMaskBits.c (revision 25870) @@ -38,7 +38,9 @@ psAssert (options->lowMask, "low mask not set"); - // mask for suspect regions due to burntool - options->burntoolMask = pmConfigMaskGet("BURNTOOL",config); - psAssert (options->burntoolMask, "burntool mask not set"); + // mask for suspect regions due to burntool if we need to. + if (options->doMaskBurntool) { + options->burntoolMask = pmConfigMaskGet("BURNTOOL",config); + psAssert (options->burntoolMask, "burntool mask not set"); + } // save MASK and MARK on the PSPHOT recipe Index: /branches/eam_branches/20090820/ppMerge/src/ppMergeLoop.c =================================================================== --- /branches/eam_branches/20090820/ppMerge/src/ppMergeLoop.c (revision 25869) +++ /branches/eam_branches/20090820/ppMerge/src/ppMergeLoop.c (revision 25870) @@ -104,6 +104,6 @@ goto ERROR; } - // the zeros vector is passed to pmReadoutCombine for each set of inputs per cell - zeros = psVectorAlloc(zeroSet->numRows, PS_TYPE_F32); + // the zeros vector is passed to pmReadoutCombine for each set of inputs per cell + zeros = psVectorAlloc(zeroSet->numRows, PS_TYPE_F32); // Flow through case PPMERGE_TYPE_FLAT: @@ -150,4 +150,21 @@ goto ERROR; } + + + // Average concepts across inputs + { + psList *inFPAs = psListAlloc(NULL); ///< List of FPAs + for (int i = 0; i < numFiles; i++) { + pmFPAfile *input = inputs->data[i]; ///< Input file + psListAdd(inFPAs, PS_LIST_TAIL, input->fpa); + } + if (!pmConceptsAverageFPAs(output->fpa, inFPAs)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average FPA concepts."); + psFree(inFPAs); + goto ERROR; + } + psFree(inFPAs); + } + pmChip *outChip; ///< Chip of interest while ((outChip = pmFPAviewNextChip(view, outFPA, 1))) { @@ -158,13 +175,29 @@ goto ERROR; } + + // Average concepts across inputs + { + psList *inChips = psListAlloc(NULL); + for (int i=0; i < numFiles; i++) { + pmChip *chip = pmFPAviewThisChip(view, ((pmFPAfile *)inputs->data[i])->fpa); + psListAdd(inChips, PS_LIST_TAIL, chip); + } + if (!pmConceptsAverageChips(outChip, inChips, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average Chip concepts."); + psFree(inChips); + goto ERROR; + } + psFree(inChips); + } + pmCell *outCell; ///< Cell of interest - // XXX TEST : force a single loop + // XXX TEST : force a single loop // outCell = pmFPAviewNextCell(view, outFPA, 1); { while ((outCell = pmFPAviewNextCell(view, outFPA, 1))) { - if (!outCell->process || !outCell->file_exists) { - continue; - } + if (!outCell->process || !outCell->file_exists) { + continue; + } if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { goto ERROR; @@ -206,15 +239,15 @@ } - if (zeroSet) { - for (int i = 0; i < zeroSet->numRows; i++) { - zeros->data.F32[i] = zeroSet->data.F32[i][cellNum]; - } - } - - int rows = psMetadataLookupS32(NULL, config->arguments, "ROWS"); // Number of rows to read per chunk - if (!rows && nThreads) { - psError(PS_ERR_UNKNOWN, false, "Invalid combination of threads > 0 and ROWS == 0 (ie, multiple threads working on the full array...)"); - goto ERROR; - } + if (zeroSet) { + for (int i = 0; i < zeroSet->numRows; i++) { + zeros->data.F32[i] = zeroSet->data.F32[i][cellNum]; + } + } + + int rows = psMetadataLookupS32(NULL, config->arguments, "ROWS"); // Number of rows to read per chunk + if (!rows && nThreads) { + psError(PS_ERR_UNKNOWN, false, "Invalid combination of threads > 0 and ROWS == 0 (ie, multiple threads working on the full array...)"); + goto ERROR; + } ppMergeFileGroup *fileGroup = NULL; @@ -227,6 +260,6 @@ pmFPAfile *input = pmFPAfileSelectSingle(config->files, "PPMERGE.INPUT", j); pmCell *inCell = pmFPAviewThisCell(view, input->fpa); ///< Input cell - pmReadout *readout = pmReadoutAlloc(inCell); - readout->process = true; // until proven otherwise, attempt to process this readout + pmReadout *readout = pmReadoutAlloc(inCell); + readout->process = true; // until proven otherwise, attempt to process this readout readouts->data[j] = readout; } @@ -361,5 +394,5 @@ psFree(fileGroups); - psFree(zeros); + psFree(zeros); // XXX eventually need to keep both the shutter and the pattern, as we do with dark @@ -424,13 +457,13 @@ // Put the new readout into the cell after the existing readouts. if (type == PPMERGE_TYPE_FRINGE && outRO) { - if (fringeSmooth) { - if (outRO->mask) { - psImage *smoothed = psImageSmoothMask (NULL, outRO->image, outRO->mask, maskVal, fringeSmoothSigma, 3, 0.2); - psFree (outRO->image); - outRO->image = smoothed; - } else { - psImageSmooth (outRO->image, fringeSmoothSigma, 3); - } - } + if (fringeSmooth) { + if (outRO->mask) { + psImage *smoothed = psImageSmoothMask (NULL, outRO->image, outRO->mask, maskVal, fringeSmoothSigma, 3, 0.2); + psFree (outRO->image); + outRO->image = smoothed; + } else { + psImageSmooth (outRO->image, fringeSmoothSigma, 3); + } + } pmFringeRegions *regions = pmFringeRegionsAlloc(fringeNum, fringeSize, fringeSize, fringeSmoothX, fringeSmoothY); @@ -447,8 +480,8 @@ // XXX replaced this : pmFringesFormat(outCell, NULL, fringes); - if (!psMetadataAdd(outCell->analysis, PS_LIST_TAIL, "FRINGE.MEASUREMENTS", PS_DATA_ARRAY, "Fringes", fringes)) { - psError(PS_ERR_UNKNOWN, false, "Unable to add fringe to analysis metadata\n"); - goto ERROR; - } + if (!psMetadataAdd(outCell->analysis, PS_LIST_TAIL, "FRINGE.MEASUREMENTS", PS_DATA_ARRAY, "Fringes", fringes)) { + psError(PS_ERR_UNKNOWN, false, "Unable to add fringe to analysis metadata\n"); + goto ERROR; + } psFree(fringes); // Drop reference } @@ -472,14 +505,14 @@ psImageMaskType cteMaskValue = pmConfigMaskGet(cteMaskName, config); - if (0) { - psFits *fits = NULL; - fits = psFitsOpen ("combine.fits", "w"); - psFitsWriteImage (fits, NULL, outRO->image, 0, NULL); - psFitsClose (fits); - - fits = psFitsOpen ("inmask.fits", "w"); - psFitsWriteImage (fits, NULL, outRO->mask, 0, NULL); - psFitsClose (fits); - } + if (0) { + psFits *fits = NULL; + fits = psFitsOpen ("combine.fits", "w"); + psFitsWriteImage (fits, NULL, outRO->image, 0, NULL); + psFitsClose (fits); + + fits = psFitsOpen ("inmask.fits", "w"); + psFitsWriteImage (fits, NULL, outRO->mask, 0, NULL); + psFitsClose (fits); + } psF32 **outputImage = outRO->image->data.F32; @@ -493,10 +526,10 @@ } - if (0) { - psFits *fits = NULL; - fits = psFitsOpen ("otmask.fits", "w"); - psFitsWriteImage (fits, NULL, outRO->mask, 0, NULL); - psFitsClose (fits); - } + if (0) { + psFits *fits = NULL; + fits = psFitsOpen ("otmask.fits", "w"); + psFitsWriteImage (fits, NULL, outRO->mask, 0, NULL); + psFitsClose (fits); + } } @@ -510,35 +543,8 @@ } - if (outChip->data_exists) { - psList *inChips = psListAlloc(NULL); - for (int i=0; i < numFiles; i++) { - pmChip *chip = pmFPAviewThisChip(view, ((pmFPAfile *)inputs->data[i])->fpa); - psListAdd(inChips, PS_LIST_TAIL, chip); - } - if (!pmConceptsAverageChips(outChip, inChips, true)) { - psError(PS_ERR_UNKNOWN, false, "Unable to average Chip concepts."); - psFree(inChips); - goto ERROR; - } - psFree(inChips); - } if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { goto ERROR; } } - - // Get list of FPAs for concepts averaging - psList *inFPAs = psListAlloc(NULL); ///< List of FPAs - for (int i = 0; i < numFiles; i++) { - pmFPAfile *input = inputs->data[i]; ///< Input file - psListAdd(inFPAs, PS_LIST_TAIL, input->fpa); - } - if (!pmConceptsAverageFPAs(output->fpa, inFPAs)) { - psError(PS_ERR_UNKNOWN, false, "Unable to average FPA concepts."); - psFree(inFPAs); - goto ERROR; - } - psFree(inFPAs); - if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { Index: /branches/eam_branches/20090820/ppStack/src/ppStackFinish.c =================================================================== --- /branches/eam_branches/20090820/ppStack/src/ppStackFinish.c (revision 25869) +++ /branches/eam_branches/20090820/ppStack/src/ppStackFinish.c (revision 25870) @@ -66,5 +66,8 @@ psString dump = psMetadataLookupStr(&mdok, config->arguments, "DUMP_CONFIG"); // File for config if (dump) { - pmConfigDump(config, dump); + if (!pmConfigDump(config, dump)) { + psError(PS_ERR_IO, false, "Unable to dump configuration."); + return false; + } } Index: /branches/eam_branches/20090820/ppSub/src/ppSub.c =================================================================== --- /branches/eam_branches/20090820/ppSub/src/ppSub.c (revision 25869) +++ /branches/eam_branches/20090820/ppSub/src/ppSub.c (revision 25870) @@ -76,4 +76,11 @@ psTimerStop(); + psString dump_file = psMetadataLookupStr(NULL, data->config->arguments, "-dumpconfig"); + if (dump_file) { + if (!pmConfigDump(data->config, dump_file)) { + psErrorStackPrint(stderr, "Unable to dump configuration.\n"); + exitValue = PS_EXIT_SYS_ERROR; + } + } psFree(data); Index: /branches/eam_branches/20090820/ppSub/src/ppSubData.c =================================================================== --- /branches/eam_branches/20090820/ppSub/src/ppSubData.c (revision 25869) +++ /branches/eam_branches/20090820/ppSub/src/ppSubData.c (revision 25870) @@ -29,8 +29,4 @@ psFree(data->stats); - psString dump_file = psMetadataLookupStr(NULL, data->config->arguments, "-dumpconfig"); - if (dump_file) { - pmConfigDump(data->config, dump_file); - } psFree(data->config); Index: /branches/eam_branches/20090820/psModules/src/camera/pmFPAfileFitsIO.c =================================================================== --- /branches/eam_branches/20090820/psModules/src/camera/pmFPAfileFitsIO.c (revision 25869) +++ /branches/eam_branches/20090820/psModules/src/camera/pmFPAfileFitsIO.c (revision 25870) @@ -179,9 +179,4 @@ pmChip *chip = pmFPAviewThisChip(view, fpa); // Chip of interest, or NULL pmCell *cell = pmFPAviewThisCell(view, fpa); // Cell of interest, or NULL - if (!pmFPAUpdateNames(fpa, chip, cell, file->imageId, file->sourceId)) { - psError(PS_ERR_UNKNOWN, false, "Unable to update names in header."); - return false; - } - if (cell) { if (!pmConceptsWriteCell(cell, true, config)) { @@ -196,4 +191,9 @@ } else if (!pmConceptsWriteFPA(fpa, true, config)) { psError(PS_ERR_IO, false, "Unable to write concepts for FPA.\n"); + return false; + } + + if (!pmFPAUpdateNames(fpa, chip, cell, file->imageId, file->sourceId)) { + psError(PS_ERR_UNKNOWN, false, "Unable to update names in header."); return false; } Index: /branches/eam_branches/20090820/psModules/src/config/pmConfigMask.c =================================================================== --- /branches/eam_branches/20090820/psModules/src/config/pmConfigMask.c (revision 25869) +++ /branches/eam_branches/20090820/psModules/src/config/pmConfigMask.c (revision 25870) @@ -16,4 +16,5 @@ { "BLANK", "DETECTOR", 0x01, true }, // Pixel doesn't contain valid data { "CTE", "DETECTOR", 0x01, true }, // Pixel has poor CTE + { "BURNTOOL", NULL, 0x04, false }, // Pixel has been touched by burntool // Invalid signal ranges { "SAT", NULL, 0x02, true }, // Pixel is saturated or non-linear Index: /branches/eam_branches/20090820/psModules/src/objects/pmPSF_IO.c =================================================================== --- /branches/eam_branches/20090820/psModules/src/objects/pmPSF_IO.c (revision 25869) +++ /branches/eam_branches/20090820/psModules/src/objects/pmPSF_IO.c (revision 25870) @@ -321,27 +321,27 @@ pmPSFClump psfClump; - // we now save clump parameters for each region : need to save all of those - int nRegions = psMetadataLookupS32 (&status, recipe, "PSF.CLUMP.NREGIONS"); - psMetadataAddS32 (header, PS_LIST_TAIL, "PSF_CLN", PS_META_REPLACE, "number of psf clump regions", nRegions); - for (int i = 0; i < nRegions; i++) { - char regionName[64]; - snprintf (regionName, 64, "PSF.CLUMP.REGION.%03d", i); - psMetadata *regionMD = psMetadataLookupPtr (&status, recipe, regionName); - - psfClump.X = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); assert (status); - psfClump.Y = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.Y"); assert (status); - psfClump.dX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DX"); assert (status); - psfClump.dY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DY"); assert (status); - - char key[16]; - snprintf (key, 9, "CLX_%03d", i); - psMetadataAddF32 (header, PS_LIST_TAIL, key, PS_META_REPLACE, "psf clump center", psfClump.X); - snprintf (key, 9, "CLY_%03d", i); - psMetadataAddF32 (header, PS_LIST_TAIL, key, PS_META_REPLACE, "psf clump center", psfClump.Y); - snprintf (key, 9, "CLDX_%03d", i); - psMetadataAddF32 (header, PS_LIST_TAIL, key, PS_META_REPLACE, "psf clump size", psfClump.dX); - snprintf (key, 9, "CLDY_%03d", i); - psMetadataAddF32 (header, PS_LIST_TAIL, key, PS_META_REPLACE, "psf clump size", psfClump.dY); - } + // we now save clump parameters for each region : need to save all of those + int nRegions = psMetadataLookupS32 (&status, analysis, "PSF.CLUMP.NREGIONS"); + psMetadataAddS32 (header, PS_LIST_TAIL, "PSF_CLN", PS_META_REPLACE, "number of psf clump regions", nRegions); + for (int i = 0; i < nRegions; i++) { + char regionName[64]; + snprintf (regionName, 64, "PSF.CLUMP.REGION.%03d", i); + psMetadata *regionMD = psMetadataLookupPtr (&status, analysis, regionName); + + psfClump.X = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); assert (status); + psfClump.Y = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.Y"); assert (status); + psfClump.dX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DX"); assert (status); + psfClump.dY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DY"); assert (status); + + char key[16]; + snprintf (key, 9, "CLX_%03d", i); + psMetadataAddF32 (header, PS_LIST_TAIL, key, PS_META_REPLACE, "psf clump center", psfClump.X); + snprintf (key, 9, "CLY_%03d", i); + psMetadataAddF32 (header, PS_LIST_TAIL, key, PS_META_REPLACE, "psf clump center", psfClump.Y); + snprintf (key, 9, "CLDX_%03d", i); + psMetadataAddF32 (header, PS_LIST_TAIL, key, PS_META_REPLACE, "psf clump size", psfClump.dX); + snprintf (key, 9, "CLDY_%03d", i); + psMetadataAddF32 (header, PS_LIST_TAIL, key, PS_META_REPLACE, "psf clump size", psfClump.dY); + } // save the dimensions of each parameter @@ -667,61 +667,61 @@ int nRegions = psMetadataLookupS32 (&status, header, "PSF_CLN"); if (!status) { - // read old-style psf clump data - - char regionName[64]; - snprintf (regionName, 64, "PSF.CLUMP.REGION.000"); - psMetadataAddS32 (recipe, PS_LIST_TAIL, "PSF.CLUMP.NREGIONS", PS_META_REPLACE, "psf clump regions", 1); - - psMetadata *regionMD = psMetadataLookupPtr (&status, recipe, regionName); - if (!regionMD) { - regionMD = psMetadataAlloc(); - psMetadataAddMetadata (recipe, PS_LIST_TAIL, regionName, PS_META_REPLACE, "psf clump region", regionMD); - psFree (regionMD); - } - - // psf clump data - pmPSFClump psfClump; - - psfClump.X = psMetadataLookupF32 (&status, header, "PSF_CLX" ); assert(status); - psfClump.Y = psMetadataLookupF32 (&status, header, "PSF_CLY" ); assert(status); - psfClump.dX = psMetadataLookupF32 (&status, header, "PSF_CLDX"); assert(status); - psfClump.dY = psMetadataLookupF32 (&status, header, "PSF_CLDY"); assert(status); - - psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.X", PS_META_REPLACE, "psf clump center", psfClump.X); - psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.Y", PS_META_REPLACE, "psf clump center", psfClump.Y); - psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DX", PS_META_REPLACE, "psf clump center", psfClump.dX); - psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DY", PS_META_REPLACE, "psf clump center", psfClump.dY); + // read old-style psf clump data + + char regionName[64]; + snprintf (regionName, 64, "PSF.CLUMP.REGION.000"); + psMetadataAddS32 (analysis, PS_LIST_TAIL, "PSF.CLUMP.NREGIONS", PS_META_REPLACE, "psf clump regions", 1); + + psMetadata *regionMD = psMetadataLookupPtr (&status, analysis, regionName); + if (!regionMD) { + regionMD = psMetadataAlloc(); + psMetadataAddMetadata (analysis, PS_LIST_TAIL, regionName, PS_META_REPLACE, "psf clump region", regionMD); + psFree (regionMD); + } + + // psf clump data + pmPSFClump psfClump; + + psfClump.X = psMetadataLookupF32 (&status, header, "PSF_CLX" ); assert(status); + psfClump.Y = psMetadataLookupF32 (&status, header, "PSF_CLY" ); assert(status); + psfClump.dX = psMetadataLookupF32 (&status, header, "PSF_CLDX"); assert(status); + psfClump.dY = psMetadataLookupF32 (&status, header, "PSF_CLDY"); assert(status); + + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.X", PS_META_REPLACE, "psf clump center", psfClump.X); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.Y", PS_META_REPLACE, "psf clump center", psfClump.Y); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DX", PS_META_REPLACE, "psf clump center", psfClump.dX); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DY", PS_META_REPLACE, "psf clump center", psfClump.dY); } else { - psMetadataAddS32 (recipe, PS_LIST_TAIL, "PSF.CLUMP.NREGIONS", PS_META_REPLACE, "psf clump regions", nRegions); - - for (int i = 0; i < nRegions; i++) { - char key[10]; - char regionName[64]; - snprintf (regionName, 64, "PSF.CLUMP.REGION.%03d", i); - - psMetadata *regionMD = psMetadataLookupPtr (&status, recipe, regionName); - if (!regionMD) { - regionMD = psMetadataAlloc(); - psMetadataAddMetadata (recipe, PS_LIST_TAIL, regionName, PS_META_REPLACE, "psf clump region", regionMD); - psFree (regionMD); - } - - // psf clump data - pmPSFClump psfClump; - - snprintf (key, 9, "CLX_%03d", i); - psfClump.X = psMetadataLookupF32 (&status, header, key); assert(status); - snprintf (key, 9, "CLY_%03d", i); - psfClump.Y = psMetadataLookupF32 (&status, header, key); assert(status); - snprintf (key, 9, "CLDX_%03d", i); - psfClump.dX = psMetadataLookupF32 (&status, header, key); assert(status); - snprintf (key, 9, "CLDY_%03d", i); - psfClump.dY = psMetadataLookupF32 (&status, header, key); assert(status); - - psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.X", PS_META_REPLACE, "psf clump center", psfClump.X); - psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.Y", PS_META_REPLACE, "psf clump center", psfClump.Y); - psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DX", PS_META_REPLACE, "psf clump center", psfClump.dX); - psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DY", PS_META_REPLACE, "psf clump center", psfClump.dY); - } + psMetadataAddS32 (recipe, PS_LIST_TAIL, "PSF.CLUMP.NREGIONS", PS_META_REPLACE, "psf clump regions", nRegions); + + for (int i = 0; i < nRegions; i++) { + char key[10]; + char regionName[64]; + snprintf (regionName, 64, "PSF.CLUMP.REGION.%03d", i); + + psMetadata *regionMD = psMetadataLookupPtr (&status, analysis, regionName); + if (!regionMD) { + regionMD = psMetadataAlloc(); + psMetadataAddMetadata (analysis, PS_LIST_TAIL, regionName, PS_META_REPLACE, "psf clump region", regionMD); + psFree (regionMD); + } + + // psf clump data + pmPSFClump psfClump; + + snprintf (key, 9, "CLX_%03d", i); + psfClump.X = psMetadataLookupF32 (&status, header, key); assert(status); + snprintf (key, 9, "CLY_%03d", i); + psfClump.Y = psMetadataLookupF32 (&status, header, key); assert(status); + snprintf (key, 9, "CLDX_%03d", i); + psfClump.dX = psMetadataLookupF32 (&status, header, key); assert(status); + snprintf (key, 9, "CLDY_%03d", i); + psfClump.dY = psMetadataLookupF32 (&status, header, key); assert(status); + + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.X", PS_META_REPLACE, "psf clump center", psfClump.X); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.Y", PS_META_REPLACE, "psf clump center", psfClump.Y); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DX", PS_META_REPLACE, "psf clump center", psfClump.dX); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DY", PS_META_REPLACE, "psf clump center", psfClump.dY); + } } @@ -871,5 +871,5 @@ psFitsReadImageBuffer(psf->residuals->Ry, file->fits, fullImage, 2); // Desired pixels } - // XXX notice that we are not saving the resid->mask + // XXX notice that we are not saving the resid->mask } Index: /branches/eam_branches/20090820/psModules/src/objects/pmPSFtryModel.c =================================================================== --- /branches/eam_branches/20090820/psModules/src/objects/pmPSFtryModel.c (revision 25869) +++ /branches/eam_branches/20090820/psModules/src/objects/pmPSFtryModel.c (revision 25870) @@ -70,16 +70,22 @@ psFree(psfTry); return NULL; - } - - int orderMin = (options->psfTrendMode == PM_TREND_MAP) ? 1 : 0; + } + + // XXX set the min number of needed source more carefully (depends on psfTrendMode?) int orderMax = PS_MAX (options->psfTrendNx, options->psfTrendNy); - - // XXX set the min number of needed source more carefully (depends on psfTrendMode?) if ((sources->n < 15) && (orderMax >= 3)) orderMax = 2; if ((sources->n < 11) && (orderMax >= 2)) orderMax = 1; if ((sources->n < 8) && (orderMax >= 1)) orderMax = 0; if ((sources->n < 3)) { - psError (PS_ERR_UNKNOWN, true, "failed to determine PSF parameters"); - return NULL; + psError (PS_ERR_UNKNOWN, true, "failed to determine PSF parameters"); + return NULL; + } + + int orderMin; + if (options->psfTrendMode == PM_TREND_MAP) { + orderMin = 1; + orderMax = PS_MAX(orderMax, 1); + } else { + orderMin = 0; } @@ -87,5 +93,5 @@ psVector *srcMask = psVectorCopy (NULL, psfTry->mask, PS_TYPE_VECTOR_MASK); - // we will save the PSF with the best fit (min systematic error) + // we will save the PSF with the best fit (min systematic error) pmPSF *minPSF = NULL; psVector *minMask = NULL; @@ -96,53 +102,53 @@ int Ny = options->psfTrendNy; for (int i = orderMin; i <= orderMax; i++) { - - if (Nx > Ny) { - options->psfTrendNx = i; - options->psfTrendNy = PS_MAX (orderMin, (int)(i * (Ny / Nx) + 0.5)); - } else { - options->psfTrendNy = i; - options->psfTrendNx = PS_MAX (orderMin, (int)(i * (Nx / Ny) + 0.5)); - } - - // free existing data, if any - psFree(psfTry->psf); - psFree(psfTry->mask); - - // allocate a mask and a psf model using the current Nx,Ny order values; - psfTry->psf = pmPSFAlloc (options); - psfTry->mask = psVectorCopy (NULL, srcMask, PS_TYPE_VECTOR_MASK); - - // stage 2: construct a psf (pmPSF) from this collection of model fits, including the 2D variation - if (!pmPSFtryMakePSF (psfTry)) { - psError(PS_ERR_UNKNOWN, false, "failed to construct a psf model from collection of sources"); - psFree(psfTry); - return NULL; - } - - // stage 3: refit with fixed shape parameters, measure pmPSFtry->metric - if (!pmPSFtryFitPSF (psfTry, options, maskVal, markVal)) { - psError(PS_ERR_UNKNOWN, false, "failed to construct a psf model from collection of sources"); - psFree(psfTry); - return NULL; - } - - // stage 4: measure systematic error in pmPSFtry->metric - if (!pmPSFtryMetric (psfTry)) { - psError(PS_ERR_UNKNOWN, false, "failed to measure systematic error of metric"); - psFree(psfTry); - return NULL; - } - - if (!minPSF) { - minPSF = psMemIncrRefCounter(psfTry->psf); - minMask = psMemIncrRefCounter(psfTry->mask); - } - - if (psfTry->psf->dApResid < minPSF->dApResid) { - psFree (minPSF); - psFree (minMask); - minPSF = psMemIncrRefCounter(psfTry->psf); - minMask = psMemIncrRefCounter(psfTry->mask); - } + + if (Nx > Ny) { + options->psfTrendNx = i; + options->psfTrendNy = PS_MAX (orderMin, (int)(i * (Ny / Nx) + 0.5)); + } else { + options->psfTrendNy = i; + options->psfTrendNx = PS_MAX (orderMin, (int)(i * (Nx / Ny) + 0.5)); + } + + // free existing data, if any + psFree(psfTry->psf); + psFree(psfTry->mask); + + // allocate a mask and a psf model using the current Nx,Ny order values; + psfTry->psf = pmPSFAlloc (options); + psfTry->mask = psVectorCopy (NULL, srcMask, PS_TYPE_VECTOR_MASK); + + // stage 2: construct a psf (pmPSF) from this collection of model fits, including the 2D variation + if (!pmPSFtryMakePSF (psfTry)) { + psError(PS_ERR_UNKNOWN, false, "failed to construct a psf model from collection of sources"); + psFree(psfTry); + return NULL; + } + + // stage 3: refit with fixed shape parameters, measure pmPSFtry->metric + if (!pmPSFtryFitPSF (psfTry, options, maskVal, markVal)) { + psError(PS_ERR_UNKNOWN, false, "failed to construct a psf model from collection of sources"); + psFree(psfTry); + return NULL; + } + + // stage 4: measure systematic error in pmPSFtry->metric + if (!pmPSFtryMetric (psfTry)) { + psError(PS_ERR_UNKNOWN, false, "failed to measure systematic error of metric"); + psFree(psfTry); + return NULL; + } + + if (!minPSF) { + minPSF = psMemIncrRefCounter(psfTry->psf); + minMask = psMemIncrRefCounter(psfTry->mask); + } + + if (psfTry->psf->dApResid < minPSF->dApResid) { + psFree (minPSF); + psFree (minMask); + minPSF = psMemIncrRefCounter(psfTry->psf); + minMask = psMemIncrRefCounter(psfTry->mask); + } } psFree (srcMask); @@ -153,5 +159,5 @@ psfTry->psf = minPSF; psfTry->mask = minMask; - + // XXXXX this is probably not used any more. Are the chisq of the fits so bad? can we // fix them by softening the errors on the brightest pixels? Index: /branches/eam_branches/20090820/psconfig/pscheckperl =================================================================== --- /branches/eam_branches/20090820/psconfig/pscheckperl (revision 25869) +++ /branches/eam_branches/20090820/psconfig/pscheckperl (revision 25870) @@ -9,14 +9,14 @@ for (; @ARGV > 0; ) { if ($ARGV[0] eq "-version") { - $version = $ARGV[1]; + $version = $ARGV[1]; shift; shift; next; } if ($ARGV[0] eq "-build") { - $build = 1; + $build = 1; shift; next; } if ($ARGV[0] eq "-force") { - $force{$ARGV[1]} = 1; - shift; shift; next; + $force{$ARGV[1]} = 1; + shift; shift; next; } if ($ARGV[0] eq "-h") { &usage (); } @@ -31,5 +31,5 @@ if ( @ARGV > 1) { &usage(); } -if ( @ARGV == 0) { +if ( @ARGV == 0) { @list = <$tagsets/*.perl>; $file = $list[-1]; @@ -79,20 +79,20 @@ ($N, $module, $tarball, $modver, $prompts) = split (" ", $line); - if ($modver eq "") { $modver = 0; } + if ($modver eq "") { $modver = 0; } system ("pscheckmods $module $modver"); - if (($? == 0) and not defined $force{$module} and not defined $force{'all'}) { - # print "$module: found\n"; - next; + if (($? == 0) and not defined $force{$module} and not defined $force{'all'}) { + # print "$module: found\n"; + next; } if (defined $force{$module}) { - print "$module: force\n"; + print "$module: force\n"; } else { - print "$module: missing\n"; + print "$module: missing\n"; } - unless ($build or defined $force{$module} or defined $force{'all'}) { - $Nmissing ++; - push @missing, $module; - next; + unless ($build or defined $force{$module} or defined $force{'all'}) { + $Nmissing ++; + push @missing, $module; + next; } @@ -102,5 +102,5 @@ print "extract $module from $tarball\n"; vsystem ("tar xvzf $tarball"); - + ($tardir) = $tarball =~ m|(\S*).tar.gz|; print "tardir: $tardir\n"; @@ -108,18 +108,24 @@ chdir $tardir; - # build the MakeMaker makefile, setting the output directories - if ($prompts) { - @answers = split (",", $prompts); - open (PIPE, "|perl Makefile.PL PREFIX=$prefix LIB=$perldir"); - foreach $answer (@answers) { - print PIPE "$answer\n"; - } - close (PIPE); + if (-e "Build.PL") { + vsystem("psbuildperl"); + vsystem("Build"); + vsystem("Build install"); } else { - vsystem ("perl Makefile.PL PREFIX=$prefix LIB=$perldir"); + # build the MakeMaker makefile, setting the output directories + if ($prompts) { + @answers = split (",", $prompts); + open (PIPE, "|perl Makefile.PL PREFIX=$prefix LIB=$perldir"); + foreach $answer (@answers) { + print PIPE "$answer\n"; + } + close (PIPE); + } else { + vsystem ("perl Makefile.PL PREFIX=$prefix LIB=$perldir"); + } + + vsystem ("make < /dev/null"); + vsystem ("make install"); } - - vsystem ("make < /dev/null"); - vsystem ("make install"); chdir $homedir; @@ -127,11 +133,11 @@ if (!$build) { if ($Nmissing > 0) { - print "The following $Nmissing perl modules are missing from your system\n"; - foreach $name (@missing) { - print " $name\n"; - } - print "you may install them in your local path by re-running pscheckperl with -build\n"; + print "The following $Nmissing perl modules are missing from your system\n"; + foreach $name (@missing) { + print " $name\n"; + } + print "you may install them in your local path by re-running pscheckperl with -build\n"; } else { - print "no perl modules are missing from your system\n"; + print "no perl modules are missing from your system\n"; } } @@ -152,7 +158,7 @@ @list = <$tagsets/*.perl>; foreach $line (@list) { - chomp $line; - ($dist) = $line =~ m|$tagsets/(\S*).perl|; - print STDERR "$dist\n"; + chomp $line; + ($dist) = $line =~ m|$tagsets/(\S*).perl|; + print STDERR "$dist\n"; } exit 2; Index: /branches/eam_branches/20090820/psconfig/tagsets/ipp-2.9.perl =================================================================== --- /branches/eam_branches/20090820/psconfig/tagsets/ipp-2.9.perl (revision 25869) +++ /branches/eam_branches/20090820/psconfig/tagsets/ipp-2.9.perl (revision 25870) @@ -2,5 +2,5 @@ 00 Getopt::Long Getopt-Long-2.36.tar.gz 2.3 n 00 Module::Build Module-Build-0.2806.tar.gz 0.2806 - 01 ExtUtils::MakeMaker ExtUtils-MakeMaker-6.31.tar.gz 0 + 01 ExtUtils::MakeMaker ExtUtils-MakeMaker-6.54.tar.gz 0 02 Params::Validate Params-Validate-0.92.tar.gz 0.92 # 02 Apache::Test Apache-Test-1.29.tar.gz 1.29 Index: /branches/eam_branches/20090820/psphot/src/psphot.h =================================================================== --- /branches/eam_branches/20090820/psphot/src/psphot.h (revision 25869) +++ /branches/eam_branches/20090820/psphot/src/psphot.h (revision 25870) @@ -182,5 +182,5 @@ bool psphotVisualShowFootprints (pmDetections *detections); bool psphotVisualShowMoments (psArray *sources); -bool psphotVisualPlotMoments (psMetadata *recipe, psArray *sources); +bool psphotVisualPlotMoments (psMetadata *recipe, psMetadata *analysis, psArray *sources); bool psphotVisualShowRoughClass (psArray *sources); bool psphotVisualShowPSFStars (psMetadata *recipe, pmPSF *psf, psArray *sources); @@ -193,5 +193,5 @@ bool psphotVisualShowResidualImage (pmReadout *readout); bool psphotVisualPlotApResid (psArray *sources, float mean, float error); -bool psphotVisualPlotSourceSize (psMetadata *recipe, psArray *sources); +bool psphotVisualPlotSourceSize (psMetadata *recipe, psMetadata *analysis, psArray *sources); bool psphotVisualShowPetrosians (psArray *sources); @@ -212,9 +212,9 @@ // bool psphotPetrosianVisualProfileRadii (psVector *radius, psVector *flux, psVector *radiusBin, psVector *fluxBin, float peakFlux, float RadiusRef); // bool psphotPetrosianVisualEllipticalContour (pmPetrosian *petrosian); -// bool psphotPetrosianVisualStats (psVector *radBin, psVector *fluxBin, -// psVector *refRadius, psVector *meanSB, -// psVector *petRatio, psVector *petRatioErr, psVector *fluxSum, -// float petRadius, float ratioForRadius, -// float petFlux, float radiusForFlux); +// bool psphotPetrosianVisualStats (psVector *radBin, psVector *fluxBin, +// psVector *refRadius, psVector *meanSB, +// psVector *petRatio, psVector *petRatioErr, psVector *fluxSum, +// float petRadius, float ratioForRadius, +// float petFlux, float radiusForFlux); bool psphotImageQuality (psMetadata *recipe, psArray *sources); Index: /branches/eam_branches/20090820/psphot/src/psphotRoughClass.c =================================================================== --- /branches/eam_branches/20090820/psphot/src/psphotRoughClass.c (revision 25869) +++ /branches/eam_branches/20090820/psphot/src/psphotRoughClass.c (revision 25870) @@ -1,11 +1,11 @@ # include "psphotInternal.h" -# define CHECK_STATUS(S,MSG) { \ - if (!status) { \ - psError(PSPHOT_ERR_CONFIG, false, "missing PSF Clump entry: %s\n", MSG); \ - return false; \ - } } +# define CHECK_STATUS(S,MSG) { \ + if (!status) { \ + psError(PSPHOT_ERR_CONFIG, false, "missing PSF Clump entry: %s\n", MSG); \ + return false; \ + } } -bool psphotRoughClassRegion (int nRegion, psRegion *region, psArray *sources, psMetadata *recipe, const bool havePSF); +bool psphotRoughClassRegion (int nRegion, psRegion *region, psArray *sources, psMetadata *target, psMetadata *recipe, const bool havePSF); // 2006.02.02 : no leaks @@ -24,16 +24,16 @@ int nRegion = 0; for (int ix = 0; ix < NX; ix ++) { - for (int iy = 0; iy < NY; iy ++) { + for (int iy = 0; iy < NY; iy ++) { - psRegion *region = psRegionAlloc (ix*dX, (ix + 1)*dX, iy*dY, (iy + 1)*dY); - if (!psphotRoughClassRegion (nRegion, region, sources, recipe, havePSF)) { - psLogMsg ("psphot", 4, "Failed to determine rough classification for region %f,%f - %f,%f\n", - region->x0, region->y0, region->x1, region->y1); - psFree (region); - continue; - } - psFree (region); - nRegion ++; - } + psRegion *region = psRegionAlloc (ix*dX, (ix + 1)*dX, iy*dY, (iy + 1)*dY); + if (!psphotRoughClassRegion (nRegion, region, sources, readout->analysis, recipe, havePSF)) { + psLogMsg ("psphot", 4, "Failed to determine rough classification for region %f,%f - %f,%f\n", + region->x0, region->y0, region->x1, region->y1); + psFree (region); + continue; + } + psFree (region); + nRegion ++; + } } psMetadataAddS32 (recipe, PS_LIST_TAIL, "PSF.CLUMP.NREGIONS", PS_META_REPLACE, "psf clump regions", nRegion); @@ -44,5 +44,5 @@ psLogMsg ("psphot.roughclass", PS_LOG_INFO, "rough classification: %f sec\n", psTimerMark ("psphot.rough")); - psphotVisualPlotMoments (recipe, sources); + psphotVisualPlotMoments (recipe, readout->analysis, sources); psphotVisualShowRoughClass (sources); // XXX better visualization: psphotVisualShowFlags (sources); @@ -51,5 +51,5 @@ } -bool psphotRoughClassRegion (int nRegion, psRegion *region, psArray *sources, psMetadata *recipe, const bool havePSF) { +bool psphotRoughClassRegion (int nRegion, psRegion *region, psArray *sources, psMetadata *target, psMetadata *recipe, const bool havePSF) { bool status; @@ -62,40 +62,40 @@ snprintf (regionName, 64, "PSF.CLUMP.REGION.%03d", nRegion); - psMetadata *regionMD = psMetadataLookupPtr (&status, recipe, regionName); + psMetadata *regionMD = psMetadataLookupPtr (&status, target, regionName); if (!regionMD) { - // allocate the region metadata folder and add this region to it. - regionMD = psMetadataAlloc(); - psMetadataAddMetadata (recipe, PS_LIST_TAIL, regionName, PS_META_REPLACE, "psf clump region", regionMD); - psFree (regionMD); + // allocate the region metadata folder and add this region to it. + regionMD = psMetadataAlloc(); + psMetadataAddMetadata (target, PS_LIST_TAIL, regionName, PS_META_REPLACE, "psf clump region", regionMD); + psFree (regionMD); } psMetadataAddPtr (regionMD, PS_LIST_TAIL, "REGION", PS_DATA_REGION | PS_META_REPLACE, "psf clump region", region); if (!havePSF) { - // determine the PSF parameters from the source moment values - // XXX why not save the psfClump as a PTR? - psfClump = pmSourcePSFClump (region, sources, recipe); - psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.X", PS_META_REPLACE, "psf clump center", psfClump.X); - psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.Y", PS_META_REPLACE, "psf clump center", psfClump.Y); - psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DX", PS_META_REPLACE, "psf clump center", psfClump.dX); - psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DY", PS_META_REPLACE, "psf clump center", psfClump.dY); + // determine the PSF parameters from the source moment values + // XXX why not save the psfClump as a PTR? + psfClump = pmSourcePSFClump (region, sources, target); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.X", PS_META_REPLACE, "psf clump center", psfClump.X); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.Y", PS_META_REPLACE, "psf clump center", psfClump.Y); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DX", PS_META_REPLACE, "psf clump center", psfClump.dX); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DY", PS_META_REPLACE, "psf clump center", psfClump.dY); } else { - // pull FWHM_X,Y from the recipe, use to define psfClump.X,Y - psfClump.X = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); - if (!status) { - psLogMsg ("psphot", 4, "No PSF clump defined for region %f,%f - %f,%f\n", region->x0, region->y0, region->x1, region->y1); - return false; - } - psfClump.Y = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.Y"); psAssert (status, "missing PSF.CLUMP.Y"); - psfClump.dX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DX"); psAssert (status, "missing PSF.CLUMP.DX"); - psfClump.dY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DY"); psAssert (status, "missing PSF.CLUMP.DY"); + // pull FWHM_X,Y from the recipe, use to define psfClump.X,Y + psfClump.X = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); + if (!status) { + psLogMsg ("psphot", 4, "No PSF clump defined for region %f,%f - %f,%f\n", region->x0, region->y0, region->x1, region->y1); + return false; + } + psfClump.Y = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.Y"); psAssert (status, "missing PSF.CLUMP.Y"); + psfClump.dX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DX"); psAssert (status, "missing PSF.CLUMP.DX"); + psfClump.dY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DY"); psAssert (status, "missing PSF.CLUMP.DY"); } if (psfClump.X < 0) { - psError(PSPHOT_ERR_PROG, false, "programming error calling pmSourcePSFClump"); - return false; + psError(PSPHOT_ERR_PROG, false, "programming error calling pmSourcePSFClump"); + return false; } if (!psfClump.X || !psfClump.Y || isnan(psfClump.X) || isnan(psfClump.Y)) { - psLogMsg ("psphot", 4, "Failed to find a valid PSF clump for region %f,%f - %f,%f\n", region->x0, region->y0, region->x1, region->y1); - return false; + psLogMsg ("psphot", 4, "Failed to find a valid PSF clump for region %f,%f - %f,%f\n", region->x0, region->y0, region->x1, region->y1); + return false; } psLogMsg ("psphot", 3, "psf clump X, Y: %f, %f\n", psfClump.X, psfClump.Y); @@ -104,6 +104,6 @@ // group into STAR, COSMIC, EXTENDED, SATURATED, etc. if (!pmSourceRoughClass (region, sources, recipe, psfClump, maskSat)) { - psError(PSPHOT_ERR_PROG, false, "programming error calling pmSourceRoughClass"); - return false; + psError(PSPHOT_ERR_PROG, false, "programming error calling pmSourceRoughClass"); + return false; } Index: /branches/eam_branches/20090820/psphot/src/psphotSourceSize.c =================================================================== --- /branches/eam_branches/20090820/psphot/src/psphotSourceSize.c (revision 25869) +++ /branches/eam_branches/20090820/psphot/src/psphotSourceSize.c (revision 25870) @@ -83,5 +83,5 @@ psLogMsg ("psphot.size", PS_LOG_INFO, "measure source sizes for %ld sources: %f sec\n", sources->n - first, psTimerMark ("psphot.size")); - psphotVisualPlotSourceSize (recipe, sources); + psphotVisualPlotSourceSize (recipe, readout->analysis, sources); psphotVisualShowSourceSize (readout, sources); psphotVisualPlotApResid (sources, options.ApResid, options.ApSysErr); @@ -103,26 +103,26 @@ pmFootprint *footprint = peak->footprint; if (!footprint) { - // if we have not footprint, use the old code to mask by isophot - psphotMaskCosmicRayIsophot (source, maskVal, crMask); - return true; + // if we have not footprint, use the old code to mask by isophot + psphotMaskCosmicRayIsophot (source, maskVal, crMask); + return true; } if (!footprint->spans) { - // if we have no footprint, use the old code to mask by isophot - psphotMaskCosmicRayIsophot (source, maskVal, crMask); - return true; + // if we have no footprint, use the old code to mask by isophot + psphotMaskCosmicRayIsophot (source, maskVal, crMask); + return true; } // mask all of the pixels covered by the spans of the footprint for (int j = 1; j < footprint->spans->n; j++) { - pmSpan *span1 = footprint->spans->data[j]; - - int iy = span1->y; - int xs = span1->x0; - int xe = span1->x1; - - for (int ix = xs; ix < xe; ix++) { - mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= crMask; - } + pmSpan *span1 = footprint->spans->data[j]; + + int iy = span1->y; + int xs = span1->x0; + int xe = span1->x1; + + for (int ix = xs; ix < xe; ix++) { + mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= crMask; + } } return true; @@ -147,14 +147,14 @@ // mark the pixels in this row to the left, then the right for (int ix = xo; ix >= 0; ix--) { - float SN = pixels->data.F32[yo][ix] / sqrt(variance->data.F32[yo][ix]); - if (SN > SN_LIMIT) { - mask->data.PS_TYPE_IMAGE_MASK_DATA[yo][ix] |= crMask; - } + float SN = pixels->data.F32[yo][ix] / sqrt(variance->data.F32[yo][ix]); + if (SN > SN_LIMIT) { + mask->data.PS_TYPE_IMAGE_MASK_DATA[yo][ix] |= crMask; + } } for (int ix = xo + 1; ix < pixels->numCols; ix++) { - float SN = pixels->data.F32[yo][ix] / sqrt(variance->data.F32[yo][ix]); - if (SN > SN_LIMIT) { - mask->data.PS_TYPE_IMAGE_MASK_DATA[yo][ix] |= crMask; - } + float SN = pixels->data.F32[yo][ix] / sqrt(variance->data.F32[yo][ix]); + if (SN > SN_LIMIT) { + mask->data.PS_TYPE_IMAGE_MASK_DATA[yo][ix] |= crMask; + } } @@ -162,33 +162,33 @@ // first go up: for (int iy = PS_MIN(yo, mask->numRows-2); iy >= 0; iy--) { - // mark the pixels in this row to the left, then the right - for (int ix = 0; ix < pixels->numCols; ix++) { - float SN = pixels->data.F32[iy][ix] / sqrt(variance->data.F32[iy][ix]); - if (SN < SN_LIMIT) continue; - - bool valid = false; - valid |= (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy+1][ix] & crMask); - valid |= (ix > 0) ? (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy+1][ix-1] & crMask) : 0; - valid |= (ix <= mask->numCols) ? (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy+1][ix+1] & crMask) : 0; - - if (!valid) continue; - mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= crMask; - } + // mark the pixels in this row to the left, then the right + for (int ix = 0; ix < pixels->numCols; ix++) { + float SN = pixels->data.F32[iy][ix] / sqrt(variance->data.F32[iy][ix]); + if (SN < SN_LIMIT) continue; + + bool valid = false; + valid |= (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy+1][ix] & crMask); + valid |= (ix > 0) ? (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy+1][ix-1] & crMask) : 0; + valid |= (ix <= mask->numCols) ? (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy+1][ix+1] & crMask) : 0; + + if (!valid) continue; + mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= crMask; + } } // next go down: for (int iy = PS_MIN(yo+1, mask->numRows-1); iy < pixels->numRows; iy++) { - // mark the pixels in this row to the left, then the right - for (int ix = 0; ix < pixels->numCols; ix++) { - float SN = pixels->data.F32[iy][ix] / sqrt(variance->data.F32[iy][ix]); - if (SN < SN_LIMIT) continue; - - bool valid = false; - valid |= (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy-1][ix] & crMask); - valid |= (ix > 0) ? (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy-1][ix-1] & crMask) : 0; - valid |= (ix <= mask->numCols) ? (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy-1][ix+1] & crMask) : 0; - - if (!valid) continue; - mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= crMask; - } + // mark the pixels in this row to the left, then the right + for (int ix = 0; ix < pixels->numCols; ix++) { + float SN = pixels->data.F32[iy][ix] / sqrt(variance->data.F32[iy][ix]); + if (SN < SN_LIMIT) continue; + + bool valid = false; + valid |= (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy-1][ix] & crMask); + valid |= (ix > 0) ? (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy-1][ix-1] & crMask) : 0; + valid |= (ix <= mask->numCols) ? (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy-1][ix+1] & crMask) : 0; + + if (!valid) continue; + mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= crMask; + } } return true; @@ -201,5 +201,5 @@ psVector *Ap = psVectorAllocEmpty (100, PS_TYPE_F32); psVector *ApErr = psVectorAllocEmpty (100, PS_TYPE_F32); - + psImageMaskType maskVal = options->maskVal | options->markVal; @@ -208,6 +208,6 @@ for (int i = 0; i < sources->n; i++) { - pmSource *source = sources->data[i]; - if (!(source->mode & PM_SOURCE_MODE_PSFSTAR)) continue; + pmSource *source = sources->data[i]; + if (!(source->mode & PM_SOURCE_MODE_PSFSTAR)) continue; // replace object in image @@ -216,22 +216,22 @@ } - // clear the mask bit and set the circular mask pixels - psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(options->markVal)); - psImageKeepCircle (source->maskObj, source->peak->x, source->peak->y, source->apRadius, "OR", options->markVal); - - // XXX can we test if psfMag is set and calculate only if needed? - pmSourceMagnitudes (source, psf, photMode, maskVal); // maskVal includes markVal - - // clear the mask bit - psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(options->markVal)); + // clear the mask bit and set the circular mask pixels + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(options->markVal)); + psImageKeepCircle (source->maskObj, source->peak->x, source->peak->y, source->apRadius, "OR", options->markVal); + + // XXX can we test if psfMag is set and calculate only if needed? + pmSourceMagnitudes (source, psf, photMode, maskVal); // maskVal includes markVal + + // clear the mask bit + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(options->markVal)); // re-subtract the object, leave local sky pmSourceSub (source, PM_MODEL_OP_FULL, options->maskVal); - float apMag = -2.5*log10(source->moments->Sum); - float dMag = source->psfMag - apMag; - - psVectorAppend (Ap, 100, dMag); - psVectorAppend (ApErr, 100, source->errMag); + float apMag = -2.5*log10(source->moments->Sum); + float dMag = source->psfMag - apMag; + + psVectorAppend (Ap, 100, dMag); + psVectorAppend (ApErr, 100, source->errMag); } @@ -242,5 +242,5 @@ psVector *dAp = psVectorAlloc (Ap->n, PS_TYPE_F32); for (int i = 0; i < Ap->n; i++) { - dAp->data.F32[i] = Ap->data.F32[i] - stats->robustMedian; + dAp->data.F32[i] = Ap->data.F32[i] - stats->robustMedian; } @@ -266,29 +266,29 @@ psLogMsg("psModules.objects", PS_LOG_INFO, "Source Size classifications: %4s %4s %4s %4s %4s %4s", "Npsf", "Next", "Nsat", "Ncr", "Nmiss", "Nskip"); - int nRegions = psMetadataLookupS32 (&status, recipe, "PSF.CLUMP.NREGIONS"); + int nRegions = psMetadataLookupS32 (&status, readout->analysis, "PSF.CLUMP.NREGIONS"); for (int i = 0; i < nRegions; i ++) { - snprintf (regionName, 64, "PSF.CLUMP.REGION.%03d", i); - psMetadata *regionMD = psMetadataLookupPtr (&status, recipe, regionName); - psAssert (regionMD, "regions must be defined by earlier call to psphotRoughClassRegion"); - - psRegion *region = psMetadataLookupPtr (&status, regionMD, "REGION"); - psAssert (region, "regions must be defined by earlier call to psphotRoughClassRegion"); - - // pull FWHM_X,Y from the recipe, use to define psfClump.X,Y - psfClump.X = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); psAssert (status, "missing PSF.CLUMP.X"); - psfClump.Y = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.Y"); psAssert (status, "missing PSF.CLUMP.Y"); - psfClump.dX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DX"); psAssert (status, "missing PSF.CLUMP.DX"); - psfClump.dY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DY"); psAssert (status, "missing PSF.CLUMP.DY"); - - if ((psfClump.X < 0) || (psfClump.Y < 0) || !psfClump.X || !psfClump.Y || isnan(psfClump.X) || isnan(psfClump.Y)) { - psLogMsg ("psphot", 4, "Failed to find a valid PSF clump for region %f,%f - %f,%f\n", region->x0, region->y0, region->x1, region->y1); - continue; - } - - if (!psphotSourceClassRegion (region, &psfClump, sources, recipe, psf, options)) { - psLogMsg ("psphot", 4, "Failed to determine source classification for region %f,%f - %f,%f\n", region->x0, region->y0, region->x1, region->y1); - continue; - } - } + snprintf (regionName, 64, "PSF.CLUMP.REGION.%03d", i); + psMetadata *regionMD = psMetadataLookupPtr (&status, readout->analysis, regionName); + psAssert (regionMD, "regions must be defined by earlier call to psphotRoughClassRegion"); + + psRegion *region = psMetadataLookupPtr (&status, regionMD, "REGION"); + psAssert (region, "regions must be defined by earlier call to psphotRoughClassRegion"); + + // pull FWHM_X,Y from the recipe, use to define psfClump.X,Y + psfClump.X = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); psAssert (status, "missing PSF.CLUMP.X"); + psfClump.Y = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.Y"); psAssert (status, "missing PSF.CLUMP.Y"); + psfClump.dX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DX"); psAssert (status, "missing PSF.CLUMP.DX"); + psfClump.dY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DY"); psAssert (status, "missing PSF.CLUMP.DY"); + + if ((psfClump.X < 0) || (psfClump.Y < 0) || !psfClump.X || !psfClump.Y || isnan(psfClump.X) || isnan(psfClump.Y)) { + psLogMsg ("psphot", 4, "Failed to find a valid PSF clump for region %f,%f - %f,%f\n", region->x0, region->y0, region->x1, region->y1); + continue; + } + + if (!psphotSourceClassRegion (region, &psfClump, sources, recipe, psf, options)) { + psLogMsg ("psphot", 4, "Failed to determine source classification for region %f,%f - %f,%f\n", region->x0, region->y0, region->x1, region->y1); + continue; + } + } return true; @@ -314,35 +314,35 @@ for (psS32 i = 0 ; i < sources->n ; i++) { - pmSource *source = (pmSource *) sources->data[i]; - - // psfClumps are found for image subregions: - // skip sources not in this region - if (source->peak->x < region->x0) continue; - if (source->peak->x >= region->x1) continue; - if (source->peak->y < region->y0) continue; - if (source->peak->y >= region->y1) continue; - - // skip source if it was already measured - if (source->tmpFlags & PM_SOURCE_TMPF_SIZE_MEASURED) { - psTrace("psphot", 7, "Not calculating source size since it has already been measured\n"); - continue; - } - - // source must have been subtracted - if (!(source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED)) { - source->mode |= PM_SOURCE_MODE_SIZE_SKIPPED; - psTrace("psphot", 7, "Not calculating source size since source is not subtracted\n"); - continue; - } - - // we are basically classifying by moments; use the default if not found - psAssert (source->moments, "why is this source missing moments?"); - if (source->mode & noMoments) { - Nskip ++; - continue; - } - - psF32 Mxx = source->moments->Mxx; - psF32 Myy = source->moments->Myy; + pmSource *source = (pmSource *) sources->data[i]; + + // psfClumps are found for image subregions: + // skip sources not in this region + if (source->peak->x < region->x0) continue; + if (source->peak->x >= region->x1) continue; + if (source->peak->y < region->y0) continue; + if (source->peak->y >= region->y1) continue; + + // skip source if it was already measured + if (source->tmpFlags & PM_SOURCE_TMPF_SIZE_MEASURED) { + psTrace("psphot", 7, "Not calculating source size since it has already been measured\n"); + continue; + } + + // source must have been subtracted + if (!(source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED)) { + source->mode |= PM_SOURCE_MODE_SIZE_SKIPPED; + psTrace("psphot", 7, "Not calculating source size since source is not subtracted\n"); + continue; + } + + // we are basically classifying by moments; use the default if not found + psAssert (source->moments, "why is this source missing moments?"); + if (source->mode & noMoments) { + Nskip ++; + continue; + } + + psF32 Mxx = source->moments->Mxx; + psF32 Myy = source->moments->Myy; // replace object in image @@ -351,58 +351,58 @@ } - // clear the mask bit and set the circular mask pixels - psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(options->markVal)); - psImageKeepCircle (source->maskObj, source->peak->x, source->peak->y, source->apRadius, "OR", options->markVal); - - // XXX can we test if psfMag is set and calculate only if needed? - pmSourceMagnitudes (source, psf, photMode, maskVal); // maskVal includes markVal - - // clear the mask bit - psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(options->markVal)); + // clear the mask bit and set the circular mask pixels + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(options->markVal)); + psImageKeepCircle (source->maskObj, source->peak->x, source->peak->y, source->apRadius, "OR", options->markVal); + + // XXX can we test if psfMag is set and calculate only if needed? + pmSourceMagnitudes (source, psf, photMode, maskVal); // maskVal includes markVal + + // clear the mask bit + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(options->markVal)); // re-subtract the object, leave local sky pmSourceSub (source, PM_MODEL_OP_FULL, options->maskVal); - float apMag = -2.5*log10(source->moments->Sum); - float dMag = source->psfMag - apMag; - float nSigma = (dMag - options->ApResid) / hypot(source->errMag, options->ApSysErr); - - source->extNsigma = nSigma; - source->tmpFlags |= PM_SOURCE_TMPF_SIZE_MEASURED; - - // Anything within this region is a probably PSF-like object. Saturated stars may land - // in this region, but are detected elsewhere on the basis of their peak value. - bool isPSF = (fabs(nSigma) < options->nSigmaApResid) && (fabs(Mxx - psfClump->X) < options->nSigmaMoments*psfClump->dX) && (fabs(Myy - psfClump->Y) < options->nSigmaMoments*psfClump->dY); - if (isPSF) { - Npsf ++; - continue; - } - - // Defects may not always match CRs from peak curvature analysis - // Defects may also be marked as SATSTAR -- XXX deactivate this flag? - // XXX this rule is not great - if ((Mxx < psfClump->X) || (Myy < psfClump->Y)) { - source->mode |= PM_SOURCE_MODE_DEFECT; - Ncr ++; - continue; - } - - // saturated star (determined in PSF fit). These may also be saturated galaxies, or - // just large saturated regions. - if (source->mode & PM_SOURCE_MODE_SATSTAR) { - Nsat ++; - continue; - } - - // XXX allow the Mxx, Myy to be less than psfClump->X,Y (by some nSigma)? - bool isEXT = (nSigma > options->nSigmaApResid) || ((Mxx > psfClump->X) && (Myy > psfClump->Y)); - if (isEXT) { - source->mode |= PM_SOURCE_MODE_EXT_LIMIT; - Next ++; - continue; - } - - psWarning ("sourse size was missed for %f,%f : %f %f -- %f\n", source->peak->xf, source->peak->yf, Mxx, Myy, nSigma); - Nmiss ++; + float apMag = -2.5*log10(source->moments->Sum); + float dMag = source->psfMag - apMag; + float nSigma = (dMag - options->ApResid) / hypot(source->errMag, options->ApSysErr); + + source->extNsigma = nSigma; + source->tmpFlags |= PM_SOURCE_TMPF_SIZE_MEASURED; + + // Anything within this region is a probably PSF-like object. Saturated stars may land + // in this region, but are detected elsewhere on the basis of their peak value. + bool isPSF = (fabs(nSigma) < options->nSigmaApResid) && (fabs(Mxx - psfClump->X) < options->nSigmaMoments*psfClump->dX) && (fabs(Myy - psfClump->Y) < options->nSigmaMoments*psfClump->dY); + if (isPSF) { + Npsf ++; + continue; + } + + // Defects may not always match CRs from peak curvature analysis + // Defects may also be marked as SATSTAR -- XXX deactivate this flag? + // XXX this rule is not great + if ((Mxx < psfClump->X) || (Myy < psfClump->Y)) { + source->mode |= PM_SOURCE_MODE_DEFECT; + Ncr ++; + continue; + } + + // saturated star (determined in PSF fit). These may also be saturated galaxies, or + // just large saturated regions. + if (source->mode & PM_SOURCE_MODE_SATSTAR) { + Nsat ++; + continue; + } + + // XXX allow the Mxx, Myy to be less than psfClump->X,Y (by some nSigma)? + bool isEXT = (nSigma > options->nSigmaApResid) || ((Mxx > psfClump->X) && (Myy > psfClump->Y)); + if (isEXT) { + source->mode |= PM_SOURCE_MODE_EXT_LIMIT; + Next ++; + continue; + } + + psWarning ("sourse size was missed for %f,%f : %f %f -- %f\n", source->peak->xf, source->peak->yf, Mxx, Myy, nSigma); + Nmiss ++; } @@ -417,5 +417,5 @@ // no longer used by psphot. float psphotModelContour(const psImage *image, const psImage *variance, const psImage *mask, - psImageMaskType maskVal, const pmModel *model, float Ro) + psImageMaskType maskVal, const pmModel *model, float Ro) { psF32 *PAR = model->params->data.F32; // Model parameters @@ -432,6 +432,6 @@ float Q = Ro * PS_SQR(sxx) / (1.0 - PS_SQR(sxx * syy * sxy) / 4.0); if (Q < 0.0) { - // ellipse is imaginary - return NAN; + // ellipse is imaginary + return NAN; } @@ -441,44 +441,44 @@ for (int x = -radius; x <= radius; x++) { - // Polynomial coefficients - // XXX Should we be using the centre of the pixel as x or x+0.5? - float A = PS_SQR (1.0 / syy); - float B = x * sxy; - float C = PS_SQR (x / sxx) - Ro; - float T = PS_SQR(B) - 4*A*C; - if (T < 0.0) { - continue; - } - - // y position in source frame - float yP = (-B + sqrt (T)) / (2.0 * A); - float yM = (-B - sqrt (T)) / (2.0 * A); - - // Get the closest pixel positions (image frame) - int xPix = x + PAR[PM_PAR_XPOS] - image->col0 + 0.5; - int yPixM = yM + PAR[PM_PAR_YPOS] - image->row0 + 0.5; - int yPixP = yP + PAR[PM_PAR_YPOS] - image->row0 + 0.5; - - if (xPix < 0 || xPix >= image->numCols) { - continue; - } - - if (yPixM >= 0 && yPixM < image->numRows && - !(mask && (mask->data.PS_TYPE_IMAGE_MASK_DATA[yPixM][xPix] & maskVal))) { - float dSigma = image->data.F32[yPixM][xPix] / sqrtf(variance->data.F32[yPixM][xPix]); - nSigma += dSigma; - nPts++; - } - - if (yPixM == yPixP) { - continue; - } - - if (yPixP >= 0 && yPixP < image->numRows && - !(mask && (mask->data.PS_TYPE_IMAGE_MASK_DATA[yPixP][xPix] & maskVal))) { - float dSigma = image->data.F32[yPixP][xPix] / sqrtf(variance->data.F32[yPixP][xPix]); - nSigma += dSigma; - nPts++; - } + // Polynomial coefficients + // XXX Should we be using the centre of the pixel as x or x+0.5? + float A = PS_SQR (1.0 / syy); + float B = x * sxy; + float C = PS_SQR (x / sxx) - Ro; + float T = PS_SQR(B) - 4*A*C; + if (T < 0.0) { + continue; + } + + // y position in source frame + float yP = (-B + sqrt (T)) / (2.0 * A); + float yM = (-B - sqrt (T)) / (2.0 * A); + + // Get the closest pixel positions (image frame) + int xPix = x + PAR[PM_PAR_XPOS] - image->col0 + 0.5; + int yPixM = yM + PAR[PM_PAR_YPOS] - image->row0 + 0.5; + int yPixP = yP + PAR[PM_PAR_YPOS] - image->row0 + 0.5; + + if (xPix < 0 || xPix >= image->numCols) { + continue; + } + + if (yPixM >= 0 && yPixM < image->numRows && + !(mask && (mask->data.PS_TYPE_IMAGE_MASK_DATA[yPixM][xPix] & maskVal))) { + float dSigma = image->data.F32[yPixM][xPix] / sqrtf(variance->data.F32[yPixM][xPix]); + nSigma += dSigma; + nPts++; + } + + if (yPixM == yPixP) { + continue; + } + + if (yPixP >= 0 && yPixP < image->numRows && + !(mask && (mask->data.PS_TYPE_IMAGE_MASK_DATA[yPixP][xPix] & maskVal))) { + float dSigma = image->data.F32[yPixP][xPix] / sqrtf(variance->data.F32[yPixP][xPix]); + nSigma += dSigma; + nPts++; + } } nSigma /= nPts; @@ -491,128 +491,128 @@ // XXX use an internal flag to mark sources which have already been measured for (int i = 0; i < sources->n; i++) { - pmSource *source = sources->data[i]; - - // skip source if it was already measured - if (source->tmpFlags & PM_SOURCE_TMPF_SIZE_MEASURED) { - psTrace("psphot", 7, "Not calculating source size since it has already been measured\n"); - continue; - } - - // source must have been subtracted - if (!(source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED)) { - source->mode |= PM_SOURCE_MODE_SIZE_SKIPPED; - psTrace("psphot", 7, "Not calculating source size since source is not subtracted\n"); - continue; - } - - psF32 **resid = source->pixels->data.F32; - psF32 **variance = source->variance->data.F32; - psImageMaskType **mask = source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA; - - // Integer position of peak - int xPeak = source->peak->xf - source->pixels->col0 + 0.5; - int yPeak = source->peak->yf - source->pixels->row0 + 0.5; - - // Skip sources which are too close to a boundary. These are mostly caught as DEFECT - if (xPeak < 1 || xPeak > source->pixels->numCols - 2 || - yPeak < 1 || yPeak > source->pixels->numRows - 2) { - psTrace("psphot", 7, "Not calculating crNsigma due to edge\n"); - continue; - } - - // Skip sources with masked pixels. These are mostly caught as DEFECT - bool keep = true; - for (int iy = -1; (iy <= +1) && keep; iy++) { - for (int ix = -1; (ix <= +1) && keep; ix++) { - if (mask[yPeak+iy][xPeak+ix] & options->maskVal) { - keep = false; - } - } - } - if (!keep) { - psTrace("psphot", 7, "Not calculating crNsigma due to masked pixels\n"); - continue; - } - - // Compare the central pixel with those on either side, for the four possible lines through it. - - // Soften variances (add systematic error) - float softening = options->soft * PS_SQR(source->peak->flux); // Softening for variances - - // Across the middle: y = 0 - float cX = 2*resid[yPeak][xPeak] - resid[yPeak+0][xPeak-1] - resid[yPeak+0][xPeak+1]; - float dcX = 4*variance[yPeak][xPeak] + variance[yPeak+0][xPeak-1] + variance[yPeak+0][xPeak+1]; - float nX = cX / sqrtf(dcX + softening); - - // Up the centre: x = 0 - float cY = 2*resid[yPeak][xPeak] - resid[yPeak-1][xPeak+0] - resid[yPeak+1][xPeak+0]; - float dcY = 4*variance[yPeak][xPeak] + variance[yPeak-1][xPeak+0] + variance[yPeak+1][xPeak+0]; - float nY = cY / sqrtf(dcY + softening); - - // Diagonal: x = y - float cL = 2*resid[yPeak][xPeak] - resid[yPeak-1][xPeak-1] - resid[yPeak+1][xPeak+1]; - float dcL = 4*variance[yPeak][xPeak] + variance[yPeak-1][xPeak-1] + variance[yPeak+1][xPeak+1]; - float nL = cL / sqrtf(dcL + softening); - - // Diagonal: x = - y - float cR = 2*resid[yPeak][xPeak] - resid[yPeak+1][xPeak-1] - resid[yPeak-1][xPeak+1]; - float dcR = 4*variance[yPeak][xPeak] + variance[yPeak+1][xPeak-1] + variance[yPeak-1][xPeak+1]; - float nR = cR / sqrtf(dcR + softening); - - // P(chisq > chisq_obs; Ndof) = gamma_Q (Ndof/2, chisq/2) - // Ndof = 4 ? (four measurements, no free parameters) - // XXX this value is going to be biased low because of systematic errors. - // we need to calibrate it somehow - // source->psfProb = gsl_sf_gamma_inc_Q (2, 0.5*chisq); - - // not strictly accurate: overcounts the chisq contribution from the center pixel (by - // factor of 4); also biases a bit low if any pixels are masked - // XXX I am not sure I want to keep this value... - source->psfChisq = PS_SQR(nX) + PS_SQR(nY) + PS_SQR(nL) + PS_SQR(nR); - - float fCR = 0.0; - int nCR = 0; - if (nX > 0.0) { - fCR += nX; - nCR ++; - } - if (nY > 0.0) { - fCR += nY; - nCR ++; - } - if (nL > 0.0) { - fCR += nL; - nCR ++; - } - if (nR > 0.0) { - fCR += nR; - nCR ++; - } - source->crNsigma = (nCR > 0) ? fCR / nCR : 0.0; - source->tmpFlags |= PM_SOURCE_TMPF_SIZE_MEASURED; - - if (!isfinite(source->crNsigma)) { - continue; - } - - // this source is thought to be a cosmic ray. flag the detection and mask the pixels - if (source->crNsigma > options->nSigmaCR) { - source->mode |= PM_SOURCE_MODE_CR_LIMIT; - // XXX still testing... : psphotMaskCosmicRay (readout->mask, source, maskVal, crMask); - // XXX acting strange... psphotMaskCosmicRay_Old (source, maskVal, crMask); - } + pmSource *source = sources->data[i]; + + // skip source if it was already measured + if (source->tmpFlags & PM_SOURCE_TMPF_SIZE_MEASURED) { + psTrace("psphot", 7, "Not calculating source size since it has already been measured\n"); + continue; + } + + // source must have been subtracted + if (!(source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED)) { + source->mode |= PM_SOURCE_MODE_SIZE_SKIPPED; + psTrace("psphot", 7, "Not calculating source size since source is not subtracted\n"); + continue; + } + + psF32 **resid = source->pixels->data.F32; + psF32 **variance = source->variance->data.F32; + psImageMaskType **mask = source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA; + + // Integer position of peak + int xPeak = source->peak->xf - source->pixels->col0 + 0.5; + int yPeak = source->peak->yf - source->pixels->row0 + 0.5; + + // Skip sources which are too close to a boundary. These are mostly caught as DEFECT + if (xPeak < 1 || xPeak > source->pixels->numCols - 2 || + yPeak < 1 || yPeak > source->pixels->numRows - 2) { + psTrace("psphot", 7, "Not calculating crNsigma due to edge\n"); + continue; + } + + // Skip sources with masked pixels. These are mostly caught as DEFECT + bool keep = true; + for (int iy = -1; (iy <= +1) && keep; iy++) { + for (int ix = -1; (ix <= +1) && keep; ix++) { + if (mask[yPeak+iy][xPeak+ix] & options->maskVal) { + keep = false; + } + } + } + if (!keep) { + psTrace("psphot", 7, "Not calculating crNsigma due to masked pixels\n"); + continue; + } + + // Compare the central pixel with those on either side, for the four possible lines through it. + + // Soften variances (add systematic error) + float softening = options->soft * PS_SQR(source->peak->flux); // Softening for variances + + // Across the middle: y = 0 + float cX = 2*resid[yPeak][xPeak] - resid[yPeak+0][xPeak-1] - resid[yPeak+0][xPeak+1]; + float dcX = 4*variance[yPeak][xPeak] + variance[yPeak+0][xPeak-1] + variance[yPeak+0][xPeak+1]; + float nX = cX / sqrtf(dcX + softening); + + // Up the centre: x = 0 + float cY = 2*resid[yPeak][xPeak] - resid[yPeak-1][xPeak+0] - resid[yPeak+1][xPeak+0]; + float dcY = 4*variance[yPeak][xPeak] + variance[yPeak-1][xPeak+0] + variance[yPeak+1][xPeak+0]; + float nY = cY / sqrtf(dcY + softening); + + // Diagonal: x = y + float cL = 2*resid[yPeak][xPeak] - resid[yPeak-1][xPeak-1] - resid[yPeak+1][xPeak+1]; + float dcL = 4*variance[yPeak][xPeak] + variance[yPeak-1][xPeak-1] + variance[yPeak+1][xPeak+1]; + float nL = cL / sqrtf(dcL + softening); + + // Diagonal: x = - y + float cR = 2*resid[yPeak][xPeak] - resid[yPeak+1][xPeak-1] - resid[yPeak-1][xPeak+1]; + float dcR = 4*variance[yPeak][xPeak] + variance[yPeak+1][xPeak-1] + variance[yPeak-1][xPeak+1]; + float nR = cR / sqrtf(dcR + softening); + + // P(chisq > chisq_obs; Ndof) = gamma_Q (Ndof/2, chisq/2) + // Ndof = 4 ? (four measurements, no free parameters) + // XXX this value is going to be biased low because of systematic errors. + // we need to calibrate it somehow + // source->psfProb = gsl_sf_gamma_inc_Q (2, 0.5*chisq); + + // not strictly accurate: overcounts the chisq contribution from the center pixel (by + // factor of 4); also biases a bit low if any pixels are masked + // XXX I am not sure I want to keep this value... + source->psfChisq = PS_SQR(nX) + PS_SQR(nY) + PS_SQR(nL) + PS_SQR(nR); + + float fCR = 0.0; + int nCR = 0; + if (nX > 0.0) { + fCR += nX; + nCR ++; + } + if (nY > 0.0) { + fCR += nY; + nCR ++; + } + if (nL > 0.0) { + fCR += nL; + nCR ++; + } + if (nR > 0.0) { + fCR += nR; + nCR ++; + } + source->crNsigma = (nCR > 0) ? fCR / nCR : 0.0; + source->tmpFlags |= PM_SOURCE_TMPF_SIZE_MEASURED; + + if (!isfinite(source->crNsigma)) { + continue; + } + + // this source is thought to be a cosmic ray. flag the detection and mask the pixels + if (source->crNsigma > options->nSigmaCR) { + source->mode |= PM_SOURCE_MODE_CR_LIMIT; + // XXX still testing... : psphotMaskCosmicRay (readout->mask, source, maskVal, crMask); + // XXX acting strange... psphotMaskCosmicRay_Old (source, maskVal, crMask); + } } // now that we have masked pixels associated with CRs, we can grow the mask if (options->grow > 0) { - bool oldThreads = psImageConvolveSetThreads(true); // Old value of threading for psImageConvolveMask - psImage *newMask = psImageConvolveMask(NULL, readout->mask, options->crMask, options->crMask, -options->grow, options->grow, -options->grow, options->grow); - psImageConvolveSetThreads(oldThreads); - if (!newMask) { - psError(PS_ERR_UNKNOWN, false, "Unable to grow CR mask"); - return false; - } - psFree(readout->mask); - readout->mask = newMask; + bool oldThreads = psImageConvolveSetThreads(true); // Old value of threading for psImageConvolveMask + psImage *newMask = psImageConvolveMask(NULL, readout->mask, options->crMask, options->crMask, -options->grow, options->grow, -options->grow, options->grow); + psImageConvolveSetThreads(oldThreads); + if (!newMask) { + psError(PS_ERR_UNKNOWN, false, "Unable to grow CR mask"); + return false; + } + psFree(readout->mask); + readout->mask = newMask; } return true; Index: /branches/eam_branches/20090820/psphot/src/psphotSourceStats.c =================================================================== --- /branches/eam_branches/20090820/psphot/src/psphotSourceStats.c (revision 25869) +++ /branches/eam_branches/20090820/psphot/src/psphotSourceStats.c (revision 25870) @@ -1,5 +1,5 @@ # include "psphotInternal.h" -bool psphotSetMomentsWindow (psMetadata *recipe, psArray *sources); +bool psphotSetMomentsWindow (psMetadata *recipe, psMetadata *analysis, psArray *sources); psArray *psphotSourceStats (pmConfig *config, pmReadout *readout, pmDetections *detections, bool setWindow) { @@ -68,8 +68,8 @@ if (setWindow) { - if (!psphotSetMomentsWindow(recipe, sources)) { - psError(PS_ERR_UNEXPECTED_NULL, false, "Failed to determine Moments Window!"); - return NULL; - } + if (!psphotSetMomentsWindow(recipe, readout->analysis, sources)) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Failed to determine Moments Window!"); + return NULL; + } } @@ -242,5 +242,5 @@ // this function attempts to iteratively determine the best value for sigma of the moments weighting Gaussian -bool psphotSetMomentsWindow (psMetadata *recipe, psArray *sources) { +bool psphotSetMomentsWindow (psMetadata *recipe, psMetadata *analysis, psArray *sources) { bool status; @@ -259,44 +259,44 @@ for (int i = 0; i < 4; i++) { - // XXX move max source number to config - for (int j = 0; (j < sources->n) && (j < 400); j++) { - - pmSource *source = sources->data[j]; - psAssert (source->moments, "force moments to exist"); - source->moments->nPixels = 0; - - // skip faint sources for moments measurement - if (source->peak->SN < MIN_SN) { - continue; - } - - // measure basic source moments (no S/N clipping on input pixels) - status = pmSourceMoments (source, 4*sigma[i], sigma[i], 0.0); - } - - // choose a grid scale that is a fixed fraction of the psf sigma^2 - psMetadataAddF32(recipe, PS_LIST_TAIL, "PSF_CLUMP_GRID_SCALE", PS_META_REPLACE, "clump grid", 0.1*PS_SQR(sigma[i])); - psMetadataAddF32(recipe, PS_LIST_TAIL, "MOMENTS_SX_MAX", PS_META_REPLACE, "moments limit", 2.0*PS_SQR(sigma[i])); - psMetadataAddF32(recipe, PS_LIST_TAIL, "MOMENTS_SY_MAX", PS_META_REPLACE, "moments limit", 2.0*PS_SQR(sigma[i])); - - // determine the PSF parameters from the source moment values - pmPSFClump psfClump = pmSourcePSFClump (NULL, sources, recipe); - psLogMsg ("psphot", 3, "radius %.1f, nStars: %d, nSigma: %5.2f, X, Y: %f, %f (%f, %f)\n", sigma[i], psfClump.nStars, psfClump.nSigma, psfClump.X, psfClump.Y, sqrt(psfClump.X) / sigma[i], sqrt(psfClump.Y) / sigma[i]); - - psMetadataAddS32 (recipe, PS_LIST_TAIL, "PSF.CLUMP.NREGIONS", PS_META_REPLACE, "psf clump regions", 1); - psMetadata *regionMD = psMetadataLookupPtr (&status, recipe, "PSF.CLUMP.REGION.000"); - if (!regionMD) { - regionMD = psMetadataAlloc(); - psMetadataAddMetadata (recipe, PS_LIST_TAIL, "PSF.CLUMP.REGION.000", PS_META_REPLACE, "psf clump region", regionMD); - psFree (regionMD); - } - psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.X", PS_META_REPLACE, "psf clump center", psfClump.X); - psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.Y", PS_META_REPLACE, "psf clump center", psfClump.Y); - psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DX", PS_META_REPLACE, "psf clump center", psfClump.dX); - psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DY", PS_META_REPLACE, "psf clump center", psfClump.dY); - - // psphotVisualPlotMoments (recipe, sources); - - Sout[i] = sqrt(0.5*(psfClump.X + psfClump.Y)) / sigma[i]; + // XXX move max source number to config + for (int j = 0; (j < sources->n) && (j < 400); j++) { + + pmSource *source = sources->data[j]; + psAssert (source->moments, "force moments to exist"); + source->moments->nPixels = 0; + + // skip faint sources for moments measurement + if (source->peak->SN < MIN_SN) { + continue; + } + + // measure basic source moments (no S/N clipping on input pixels) + status = pmSourceMoments (source, 4*sigma[i], sigma[i], 0.0); + } + + // choose a grid scale that is a fixed fraction of the psf sigma^2 + psMetadataAddF32(recipe, PS_LIST_TAIL, "PSF_CLUMP_GRID_SCALE", PS_META_REPLACE, "clump grid", 0.1*PS_SQR(sigma[i])); + psMetadataAddF32(recipe, PS_LIST_TAIL, "MOMENTS_SX_MAX", PS_META_REPLACE, "moments limit", 2.0*PS_SQR(sigma[i])); + psMetadataAddF32(recipe, PS_LIST_TAIL, "MOMENTS_SY_MAX", PS_META_REPLACE, "moments limit", 2.0*PS_SQR(sigma[i])); + + // determine the PSF parameters from the source moment values + pmPSFClump psfClump = pmSourcePSFClump (NULL, sources, recipe); + psLogMsg ("psphot", 3, "radius %.1f, nStars: %d, nSigma: %5.2f, X, Y: %f, %f (%f, %f)\n", sigma[i], psfClump.nStars, psfClump.nSigma, psfClump.X, psfClump.Y, sqrt(psfClump.X) / sigma[i], sqrt(psfClump.Y) / sigma[i]); + + psMetadataAddS32 (analysis, PS_LIST_TAIL, "PSF.CLUMP.NREGIONS", PS_META_REPLACE, "psf clump regions", 1); + psMetadata *regionMD = psMetadataLookupPtr (&status, analysis, "PSF.CLUMP.REGION.000"); + if (!regionMD) { + regionMD = psMetadataAlloc(); + psMetadataAddMetadata (analysis, PS_LIST_TAIL, "PSF.CLUMP.REGION.000", PS_META_REPLACE, "psf clump region", regionMD); + psFree (regionMD); + } + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.X", PS_META_REPLACE, "psf clump center", psfClump.X); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.Y", PS_META_REPLACE, "psf clump center", psfClump.Y); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DX", PS_META_REPLACE, "psf clump center", psfClump.dX); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DY", PS_META_REPLACE, "psf clump center", psfClump.dY); + + // psphotVisualPlotMoments (recipe, sources); + + Sout[i] = sqrt(0.5*(psfClump.X + psfClump.Y)) / sigma[i]; } @@ -307,6 +307,6 @@ float maxS = Sout[0]; for (int i = 0; i < 4; i++) { - minS = PS_MIN(Sout[i], minS); - maxS = PS_MAX(Sout[i], maxS); + minS = PS_MIN(Sout[i], minS); + maxS = PS_MAX(Sout[i], maxS); } if (minS > 0.65) Sigma = sigma[3]; @@ -314,10 +314,10 @@ for (int i = 0; i < 3; i++) { - if ((Sout[i] > 0.65) && (Sout[i+1] > 0.65)) continue; - if ((Sout[i] < 0.65) && (Sout[i+1] < 0.65)) continue; - Sigma = sigma[i] + (0.65 - Sout[i])*(sigma[i+1] - sigma[i])/(Sout[i+1] - Sout[i]); + if ((Sout[i] > 0.65) && (Sout[i+1] > 0.65)) continue; + if ((Sout[i] < 0.65) && (Sout[i+1] < 0.65)) continue; + Sigma = sigma[i] + (0.65 - Sout[i])*(sigma[i+1] - sigma[i])/(Sout[i+1] - Sout[i]); } psAssert (isfinite(Sigma), "did we miss a case?"); - + // choose a grid scale that is a fixed fraction of the psf sigma^2 psMetadataAddF32(recipe, PS_LIST_TAIL, "PSF_CLUMP_GRID_SCALE", PS_META_REPLACE, "clump grid", 0.1*PS_SQR(Sigma)); Index: /branches/eam_branches/20090820/psphot/src/psphotVisual.c =================================================================== --- /branches/eam_branches/20090820/psphot/src/psphotVisual.c (revision 25869) +++ /branches/eam_branches/20090820/psphot/src/psphotVisual.c (revision 25870) @@ -28,32 +28,32 @@ switch (channel) { case 1: - if (kapa1 == -1) { - kapa1 = KapaOpenNamedSocket ("kapa", "psphot:images"); - if (kapa1 == -1) { - fprintf (stderr, "failure to open kapa; visual mode disabled\n"); - pmVisualSetVisual(false); - } - } - return kapa1; + if (kapa1 == -1) { + kapa1 = KapaOpenNamedSocket ("kapa", "psphot:images"); + if (kapa1 == -1) { + fprintf (stderr, "failure to open kapa; visual mode disabled\n"); + pmVisualSetVisual(false); + } + } + return kapa1; case 2: - if (kapa2 == -1) { - kapa2 = KapaOpenNamedSocket ("kapa", "psphot:plots"); - if (kapa2 == -1) { - fprintf (stderr, "failure to open kapa; visual mode disabled\n"); - pmVisualSetVisual(false); - } - } - return kapa2; + if (kapa2 == -1) { + kapa2 = KapaOpenNamedSocket ("kapa", "psphot:plots"); + if (kapa2 == -1) { + fprintf (stderr, "failure to open kapa; visual mode disabled\n"); + pmVisualSetVisual(false); + } + } + return kapa2; case 3: - if (kapa3 == -1) { - kapa3 = KapaOpenNamedSocket ("kapa", "psphot:stamps"); - if (kapa3 == -1) { - fprintf (stderr, "failure to open kapa; visual mode disabled\n"); - pmVisualSetVisual(false); - } - } - return kapa3; + if (kapa3 == -1) { + kapa3 = KapaOpenNamedSocket ("kapa", "psphot:stamps"); + if (kapa3 == -1) { + fprintf (stderr, "failure to open kapa; visual mode disabled\n"); + pmVisualSetVisual(false); + } + } + return kapa3; default: - psAbort ("unknown kapa channel"); + psAbort ("unknown kapa channel"); } psAbort ("unknown kapa channel"); @@ -315,5 +315,5 @@ // draw the top - // XXX need to allow top (and bottom) to have more than one span + // XXX need to allow top (and bottom) to have more than one span span = footprint->spans->data[0]; overlay[Noverlay].type = KII_OVERLAY_LINE; @@ -448,5 +448,5 @@ } -bool psphotVisualPlotMoments (psMetadata *recipe, psArray *sources) { +bool psphotVisualPlotMoments (psMetadata *recipe, psMetadata *analysis, psArray *sources) { bool status; @@ -469,29 +469,29 @@ float Ymin = 1000.0, Ymax = 0.0; { - int nRegions = psMetadataLookupS32 (&status, recipe, "PSF.CLUMP.NREGIONS"); + int nRegions = psMetadataLookupS32 (&status, analysis, "PSF.CLUMP.NREGIONS"); for (int n = 0; n < nRegions; n++) { char regionName[64]; snprintf (regionName, 64, "PSF.CLUMP.REGION.%03d", n); - psMetadata *regionMD = psMetadataLookupPtr (&status, recipe, regionName); - - float psfX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); + psMetadata *regionMD = psMetadataLookupPtr (&status, analysis, regionName); + + float psfX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); float psfY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.Y"); float psfdX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DX"); float psfdY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DY"); - float X0 = psfX - 4.0*psfdX; - float X1 = psfX + 4.0*psfdX; - float Y0 = psfY - 4.0*psfdY; - float Y1 = psfY + 4.0*psfdY; - - if (isfinite(X0)) { Xmin = PS_MIN(Xmin, X0); } - if (isfinite(X1)) { Xmax = PS_MAX(Xmax, X1); } - if (isfinite(Y0)) { Ymin = PS_MIN(Ymin, Y0); } - if (isfinite(Y1)) { Ymax = PS_MAX(Ymax, Y1); } - } - } - Xmin = PS_MAX(Xmin, -0.1); - Ymin = PS_MAX(Ymin, -0.1); + float X0 = psfX - 4.0*psfdX; + float X1 = psfX + 4.0*psfdX; + float Y0 = psfY - 4.0*psfdY; + float Y1 = psfY + 4.0*psfdY; + + if (isfinite(X0)) { Xmin = PS_MIN(Xmin, X0); } + if (isfinite(X1)) { Xmax = PS_MAX(Xmax, X1); } + if (isfinite(Y0)) { Ymin = PS_MIN(Ymin, Y0); } + if (isfinite(Y1)) { Ymax = PS_MAX(Ymax, Y1); } + } + } + Xmin = PS_MAX(Xmin, -0.1); + Ymin = PS_MAX(Ymin, -0.1); // storage vectors for data to be plotted @@ -653,15 +653,15 @@ psVector *yLimit = psVectorAlloc (120, PS_TYPE_F32); - int nRegions = psMetadataLookupS32 (&status, recipe, "PSF.CLUMP.NREGIONS"); - float PSF_CLUMP_NSIGMA = psMetadataLookupF32 (&status, recipe, "PSF_CLUMP_NSIGMA"); + int nRegions = psMetadataLookupS32 (&status, analysis, "PSF.CLUMP.NREGIONS"); + float PSF_CLUMP_NSIGMA = psMetadataLookupF32 (&status, analysis, "PSF_CLUMP_NSIGMA"); graphdata.color = KapaColorByName ("blue"); graphdata.style = 0; - graphdata.xmin = Xmin; - graphdata.ymin = Ymin; - graphdata.xmax = Xmax; - graphdata.ymax = Ymax; - KapaSetLimits (myKapa, &graphdata); + graphdata.xmin = Xmin; + graphdata.ymin = Ymin; + graphdata.xmax = Xmax; + graphdata.ymax = Ymax; + KapaSetLimits (myKapa, &graphdata); for (int n = 0; n < nRegions; n++) { @@ -669,5 +669,5 @@ char regionName[64]; snprintf (regionName, 64, "PSF.CLUMP.REGION.%03d", n); - psMetadata *regionMD = psMetadataLookupPtr (&status, recipe, regionName); + psMetadata *regionMD = psMetadataLookupPtr (&status, analysis, regionName); float psfX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); @@ -926,12 +926,12 @@ if (Xo == 0) { // place source alone on this row - bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); - if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); + if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); psphotMosaicSubimage (outpos, source, Xo, Yo, DX, DY, true); - pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); psphotMosaicSubimage (outsub, source, Xo, Yo, DX, DY, true); - if (!subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + if (!subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); Yo += DY; @@ -943,12 +943,12 @@ Xo = 0; - bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); - if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); + if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); psphotMosaicSubimage (outpos, source, Xo, Yo, DX, DY, true); - pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); psphotMosaicSubimage (outsub, source, Xo, Yo, DX, DY, true); - if (!subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + if (!subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); Xo = DX; @@ -957,11 +957,11 @@ } else { // extend this row - bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); - if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); + if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); psphotMosaicSubimage (outpos, source, Xo, Yo, DX, DY, true); - pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); psphotMosaicSubimage (outsub, source, Xo, Yo, DX, DY, true); - if (!subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + if (!subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); Xo += DX; @@ -1020,5 +1020,5 @@ pmSource *source = sources->data[i]; - // only show "real" saturated stars (not defects) + // only show "real" saturated stars (not defects) if (!(source->mode & PM_SOURCE_MODE_SATSTAR)) continue;; if (source->mode & PM_SOURCE_MODE_DEFECT) continue;; @@ -1065,8 +1065,8 @@ pmSource *source = sources->data[i]; - // only show "real" saturated stars (not defects) + // only show "real" saturated stars (not defects) if (!(source->mode & PM_SOURCE_MODE_SATSTAR)) continue;; if (source->mode & PM_SOURCE_MODE_DEFECT) continue;; - nSAT ++; + nSAT ++; if (Xo + DX > NX) { @@ -1074,8 +1074,8 @@ if (Xo == 0) { // place source alone on this row - bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); - if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); + if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); psphotMosaicSubimage (outsat, source, Xo, Yo, DX, DY, false); - if (subtracted) pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + if (subtracted) pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); Yo += DY; @@ -1087,8 +1087,8 @@ Xo = 0; - bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); - if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); + if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); psphotMosaicSubimage (outsat, source, Xo, Yo, DX, DY, false); - if (subtracted) pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + if (subtracted) pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); Xo = DX; @@ -1097,8 +1097,8 @@ } else { // extend this row - bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); - if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); + if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); psphotMosaicSubimage (outsat, source, Xo, Yo, DX, DY, false); - if (subtracted) pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + if (subtracted) pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); Xo += DX; @@ -1406,11 +1406,11 @@ // if (source->type != type) continue; - if (mode) { - if (keep) { - if (!(source->mode & mode)) continue; - } else { - if (source->mode & mode) continue; - } - } + if (mode) { + if (keep) { + if (!(source->mode & mode)) continue; + } else { + if (source->mode & mode) continue; + } + } pmMoments *moments = source->moments; @@ -1463,5 +1463,5 @@ } -bool psphotVisualPlotSourceSize (psMetadata *recipe, psArray *sources) { +bool psphotVisualPlotSourceSize (psMetadata *recipe, psMetadata *analysis, psArray *sources) { bool status; @@ -1482,29 +1482,29 @@ float Ymin = 1000.0, Ymax = 0.0; { - int nRegions = psMetadataLookupS32 (&status, recipe, "PSF.CLUMP.NREGIONS"); + int nRegions = psMetadataLookupS32 (&status, analysis, "PSF.CLUMP.NREGIONS"); for (int n = 0; n < nRegions; n++) { char regionName[64]; snprintf (regionName, 64, "PSF.CLUMP.REGION.%03d", n); - psMetadata *regionMD = psMetadataLookupPtr (&status, recipe, regionName); - - float psfX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); + psMetadata *regionMD = psMetadataLookupPtr (&status, analysis, regionName); + + float psfX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); float psfY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.Y"); float psfdX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DX"); float psfdY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DY"); - float X0 = psfX - 10.0*psfdX; - float X1 = psfX + 10.0*psfdX; - float Y0 = psfY - 10.0*psfdY; - float Y1 = psfY + 10.0*psfdY; - - if (isfinite(X0)) { Xmin = PS_MIN(Xmin, X0); } - if (isfinite(X1)) { Xmax = PS_MAX(Xmax, X1); } - if (isfinite(Y0)) { Ymin = PS_MIN(Ymin, Y0); } - if (isfinite(Y1)) { Ymax = PS_MAX(Ymax, Y1); } - } - } - Xmin = PS_MAX(Xmin, -0.1); - Ymin = PS_MAX(Ymin, -0.1); + float X0 = psfX - 10.0*psfdX; + float X1 = psfX + 10.0*psfdX; + float Y0 = psfY - 10.0*psfdY; + float Y1 = psfY + 10.0*psfdY; + + if (isfinite(X0)) { Xmin = PS_MIN(Xmin, X0); } + if (isfinite(X1)) { Xmax = PS_MAX(Xmax, X1); } + if (isfinite(Y0)) { Ymin = PS_MIN(Ymin, Y0); } + if (isfinite(Y1)) { Ymax = PS_MAX(Ymax, Y1); } + } + } + Xmin = PS_MAX(Xmin, -0.1); + Ymin = PS_MAX(Ymin, -0.1); // storage vectors for data to be plotted @@ -1544,42 +1544,42 @@ if (source->moments == NULL) continue; - if (source->mode & PM_SOURCE_MODE_CR_LIMIT) { - xCR->data.F32[nCR] = source->moments->Mxx; - yCR->data.F32[nCR] = source->moments->Myy; - mCR->data.F32[nCR] = -2.5*log10(source->moments->Sum); - sCR->data.F32[nCR] = source->extNsigma; - nCR++; - } - if (source->mode & PM_SOURCE_MODE_SATSTAR) { - xSAT->data.F32[nSAT] = source->moments->Mxx; - ySAT->data.F32[nSAT] = source->moments->Myy; - mSAT->data.F32[nSAT] = -2.5*log10(source->moments->Sum); - sSAT->data.F32[nSAT] = source->extNsigma; - nSAT++; - } - if (source->mode & PM_SOURCE_MODE_EXT_LIMIT) { - xEXT->data.F32[nEXT] = source->moments->Mxx; - yEXT->data.F32[nEXT] = source->moments->Myy; - mEXT->data.F32[nEXT] = -2.5*log10(source->moments->Sum); - sEXT->data.F32[nEXT] = source->extNsigma; - nEXT++; - continue; - } - if (source->mode & PM_SOURCE_MODE_DEFECT) { - xDEF->data.F32[nDEF] = source->moments->Mxx; - yDEF->data.F32[nDEF] = source->moments->Myy; - mDEF->data.F32[nDEF] = -2.5*log10(source->moments->Sum); - sDEF->data.F32[nDEF] = source->extNsigma; - nDEF++; - continue; - } - if ((source->mode & PM_SOURCE_MODE_CR_LIMIT) || (source->mode & PM_SOURCE_MODE_SATSTAR)) { - continue; - } - xPSF->data.F32[nPSF] = source->moments->Mxx; - yPSF->data.F32[nPSF] = source->moments->Myy; - mPSF->data.F32[nPSF] = -2.5*log10(source->moments->Sum); - sPSF->data.F32[nPSF] = source->extNsigma; - nPSF++; + if (source->mode & PM_SOURCE_MODE_CR_LIMIT) { + xCR->data.F32[nCR] = source->moments->Mxx; + yCR->data.F32[nCR] = source->moments->Myy; + mCR->data.F32[nCR] = -2.5*log10(source->moments->Sum); + sCR->data.F32[nCR] = source->extNsigma; + nCR++; + } + if (source->mode & PM_SOURCE_MODE_SATSTAR) { + xSAT->data.F32[nSAT] = source->moments->Mxx; + ySAT->data.F32[nSAT] = source->moments->Myy; + mSAT->data.F32[nSAT] = -2.5*log10(source->moments->Sum); + sSAT->data.F32[nSAT] = source->extNsigma; + nSAT++; + } + if (source->mode & PM_SOURCE_MODE_EXT_LIMIT) { + xEXT->data.F32[nEXT] = source->moments->Mxx; + yEXT->data.F32[nEXT] = source->moments->Myy; + mEXT->data.F32[nEXT] = -2.5*log10(source->moments->Sum); + sEXT->data.F32[nEXT] = source->extNsigma; + nEXT++; + continue; + } + if (source->mode & PM_SOURCE_MODE_DEFECT) { + xDEF->data.F32[nDEF] = source->moments->Mxx; + yDEF->data.F32[nDEF] = source->moments->Myy; + mDEF->data.F32[nDEF] = -2.5*log10(source->moments->Sum); + sDEF->data.F32[nDEF] = source->extNsigma; + nDEF++; + continue; + } + if ((source->mode & PM_SOURCE_MODE_CR_LIMIT) || (source->mode & PM_SOURCE_MODE_SATSTAR)) { + continue; + } + xPSF->data.F32[nPSF] = source->moments->Mxx; + yPSF->data.F32[nPSF] = source->moments->Myy; + mPSF->data.F32[nPSF] = -2.5*log10(source->moments->Sum); + sPSF->data.F32[nPSF] = source->extNsigma; + nPSF++; } xSAT->n = nSAT; @@ -1861,15 +1861,15 @@ psVector *yLimit = psVectorAlloc (120, PS_TYPE_F32); - int nRegions = psMetadataLookupS32 (&status, recipe, "PSF.CLUMP.NREGIONS"); - float PSF_CLUMP_NSIGMA = psMetadataLookupF32 (&status, recipe, "PSF_CLUMP_NSIGMA"); + int nRegions = psMetadataLookupS32 (&status, analysis, "PSF.CLUMP.NREGIONS"); + float PSF_CLUMP_NSIGMA = psMetadataLookupF32 (&status, analysis, "PSF_CLUMP_NSIGMA"); graphdata.color = KapaColorByName ("blue"); graphdata.style = 0; - graphdata.xmin = Xmin; - graphdata.ymin = Ymin; - graphdata.xmax = Xmax; - graphdata.ymax = Ymax; - KapaSetLimits (myKapa, &graphdata); + graphdata.xmin = Xmin; + graphdata.ymin = Ymin; + graphdata.xmax = Xmax; + graphdata.ymax = Ymax; + KapaSetLimits (myKapa, &graphdata); for (int n = 0; n < nRegions; n++) { @@ -1877,5 +1877,5 @@ char regionName[64]; snprintf (regionName, 64, "PSF.CLUMP.REGION.%03d", n); - psMetadata *regionMD = psMetadataLookupPtr (&status, recipe, regionName); + psMetadata *regionMD = psMetadataLookupPtr (&status, analysis, regionName); float psfX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); @@ -2069,32 +2069,32 @@ for (int i = 0; i < sources->n; i++) { - pmSource *source = sources->data[i]; - - if (!source) continue; - if (!source->extpars) continue; - if (!source->extpars->profile) continue; - if (!source->extpars->petrosian_80) continue; - - pmSourceRadialProfile *profile = source->extpars->profile; - pmSourceExtendedFlux *petrosian = source->extpars->petrosian_80; - - overlay[Noverlay].type = KII_OVERLAY_CIRCLE; - overlay[Noverlay].x = source->peak->xf; - overlay[Noverlay].y = source->peak->yf; - overlay[Noverlay].dx = 2.0*petrosian->radius; - overlay[Noverlay].dy = 2.0*petrosian->radius*profile->axes.minor/profile->axes.major; - overlay[Noverlay].angle = profile->axes.theta * PS_DEG_RAD; - overlay[Noverlay].text = NULL; - Noverlay ++; + pmSource *source = sources->data[i]; + + if (!source) continue; + if (!source->extpars) continue; + if (!source->extpars->profile) continue; + if (!source->extpars->petrosian_80) continue; + + pmSourceRadialProfile *profile = source->extpars->profile; + pmSourceExtendedFlux *petrosian = source->extpars->petrosian_80; + + overlay[Noverlay].type = KII_OVERLAY_CIRCLE; + overlay[Noverlay].x = source->peak->xf; + overlay[Noverlay].y = source->peak->yf; + overlay[Noverlay].dx = 2.0*petrosian->radius; + overlay[Noverlay].dy = 2.0*petrosian->radius*profile->axes.minor/profile->axes.major; + overlay[Noverlay].angle = profile->axes.theta * PS_DEG_RAD; + overlay[Noverlay].text = NULL; + Noverlay ++; CHECK_REALLOCATE (overlay, KiiOverlay, NOVERLAY, Noverlay, 100); - overlay[Noverlay].type = KII_OVERLAY_CIRCLE; - overlay[Noverlay].x = source->peak->xf; - overlay[Noverlay].y = source->peak->yf; - overlay[Noverlay].dx = 4.0*petrosian->radius; - overlay[Noverlay].dy = 4.0*petrosian->radius*profile->axes.minor/profile->axes.major; - overlay[Noverlay].angle = profile->axes.theta * PS_DEG_RAD; - overlay[Noverlay].text = NULL; - Noverlay ++; + overlay[Noverlay].type = KII_OVERLAY_CIRCLE; + overlay[Noverlay].x = source->peak->xf; + overlay[Noverlay].y = source->peak->yf; + overlay[Noverlay].dx = 4.0*petrosian->radius; + overlay[Noverlay].dy = 4.0*petrosian->radius*profile->axes.minor/profile->axes.major; + overlay[Noverlay].angle = profile->axes.theta * PS_DEG_RAD; + overlay[Noverlay].text = NULL; + Noverlay ++; CHECK_REALLOCATE (overlay, KiiOverlay, NOVERLAY, Noverlay, 100); } Index: /branches/eam_branches/20090820/pstamp/scripts/pstampparse.pl =================================================================== --- /branches/eam_branches/20090820/pstamp/scripts/pstampparse.pl (revision 25869) +++ /branches/eam_branches/20090820/pstamp/scripts/pstampparse.pl (revision 25870) @@ -72,4 +72,8 @@ } +# just deal with these arguments once and for all +$pstamptool .= " -dbname $dbname" if $dbname; +$pstamptool .= " -dbserver $dbserver" if $dbserver; + # list_job is a deugging mode $no_update = 1 if $mode eq "list_job"; @@ -105,6 +109,4 @@ if ($req_id and !$no_update) { my $command = "$pstamptool -listreq -name $req_name -not_req_id $req_id"; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; # no verbose so that error message about request not found doesn't appear in parse_error.txt my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -127,6 +129,4 @@ my $command = "$pstamptool -updatereq -req_id $req_id -name $req_name"; $command .= " -outProduct $product"; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -195,4 +195,6 @@ my $mjd_max = $row->{MJD_MAX}; my $label = $row->{LABEL}; + my $x = $row->{CENTER_X}; + my $y = $row->{CENTER_Y}; my $option_mask= $row->{OPTION_MASK}; @@ -205,5 +207,5 @@ if (!$skycenter and !$search_component) { - print STDERR "COMPONENT must be specified for pixel coordinate ROI\n" if $verbose; + print STDERR "COMPONENT must be specified for pixel coordinate ROI center\n" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); $num_jobs++; @@ -213,5 +215,5 @@ $search_component = "" if $search_component eq "all"; - if (($job_type eq "stamp") and ! validROI($row)) { + if ((($job_type eq "stamp") or ($req_type eq "bycoord")) and ! validROI($row)) { print STDERR "invalid ROI for row $rownum\n" if $verbose; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); @@ -262,23 +264,22 @@ next; } - print STDERR "lookup bycoord is not yet implemented" if $verbose; - insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED); - $num_jobs++; - next; - } else { - # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this - # request specification. An array reference is returned. - my ($x, $y); - - $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $tess_id, $search_component, + if ($stage eq "stack") { + print STDERR "lookup bycoord is not yet implemented for stage stack" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED); + $num_jobs++; + next; + } + } + # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this + # request specification. An array reference is returned. + $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $tess_id, $search_component, $inverse, $need_magic, $x, $y, $mjd_min, $mjd_max, $filter, $label, $verbose); - if (!$imageList or !@$imageList) { - print STDERR "no matching images found for row $rownum\n" if $verbose; - # note in this case queueJobs inserts the fake job for these rows - } - $row->{need_magic} = $need_magic; - push @rowList, $row; - } + if (!$imageList or !@$imageList) { + print STDERR "no matching images found for row $rownum\n" if $verbose; + # note in this case queueJobs inserts the fake job for these rows + } + $row->{need_magic} = $need_magic; + push @rowList, $row; } @@ -397,14 +398,26 @@ my $newState = "run"; my $fault = 0; - - if (($stage ne 'stack') and ($stage ne 'raw')) { - if (($image->{state} eq 'goto_purged') or ($image->{data_state} eq 'purged')) { + my $dep_id; + + if ($stage ne 'raw') { + my $run_state = $image->{state}; + my $data_state = $image->{data_state}; + $data_state = $run_state if $stage eq "stack"; + if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or + ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) { + # image is gone and it's not coming back $newState = 'stop'; $fault = $PSTAMP_GONE; - } elsif (($image->{data_state} ne 'full') or ($image->{state} ne 'full' )) { - # XXX here is where we need to queue an update job - # for now just say that the image is not available - $newState = 'stop'; - $fault = $PSTAMP_NOT_AVAILABLE; + } elsif (($data_state ne 'full') or ($run_state ne 'full' )) { + # don't wait for update unless the caller asks us to + if (!($option_mask & $PSTAMP_WAIT_FOR_UPDATE)) { + $newState = 'stop'; + $fault = $PSTAMP_NOT_AVAILABLE; + } else { + # cause the image to be re-made + # set up to queue an update run + queue_update_run(\$newState, \$fault, \$dep_id, $image->{image_db}, + $run_state, $stage, $image->{stage_id}, $need_magic, $image->{label}); + } } } @@ -415,6 +428,5 @@ $command .= " -fault $fault" if $fault; $command .= " -exp_id $exp_id" if $exp_id; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; + $command .= " -dep_id $dep_id" if $dep_id; if ($mode eq "list_job") { @@ -593,6 +605,4 @@ my $command = "$pstamptool -addjob -req_id $req_id -job_type $job_type" . " -rownum $rownum -state stop -fault $fault"; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; if ($mode eq "list_job") { @@ -717,4 +727,36 @@ } +sub queue_update_run +{ + my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $need_magic, $label) = @_; + + if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned')) { + my_die("$stage $stage_id is in unexpected state $state", $PS_EXIT_PROG_ERROR); + } + + my $dep_id; + my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb"; + $command .= " -rlabel $label" if $label; + $command .= " -no_magic" if !$need_magic; + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR); + } + my $output = join "", @$stdout_buf; + chomp $output; + $dep_id = $output; + my_die("pstamptool -getdependent returned invalid dep_id", $PS_EXIT_PROG_ERROR) if !$dep_id; + } else { + print STDERR "skipping $command\n"; + $dep_id = 42; + } + + $$r_dep_id = $dep_id; + $r_fault = 0; + $r_jobState = 'blocked'; +} + sub my_die { Index: /branches/eam_branches/20090820/pstamp/src/ppstampMakeStamp.c =================================================================== --- /branches/eam_branches/20090820/pstamp/src/ppstampMakeStamp.c (revision 25869) +++ /branches/eam_branches/20090820/pstamp/src/ppstampMakeStamp.c (revision 25870) @@ -157,4 +157,7 @@ return NULL; } + if (copyWidth > width) { + copyWidth = width; + } psImage *output = psImageAlloc(width, height, image->type.type); Index: /branches/eam_branches/20090820/pstamp/test/byskycell.txt =================================================================== --- /branches/eam_branches/20090820/pstamp/test/byskycell.txt (revision 25869) +++ /branches/eam_branches/20090820/pstamp/test/byskycell.txt (revision 25870) @@ -15,14 +15,6 @@ # subsequent lines define the rows in the table -################################ OLD Format -# -# ROWNUM PROJECT JOB_TYPE OPTION_MASK REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT COORD_MASK CENTER_X CENTER_Y WIDTH HEIGHT REQFILT MJD_MIN MJD_MAX -# warps from various epochs for one of the SN candidates. Specifiying the skycell speeds up processing -# 1 gpc1 stamp 1 byskycell warp null MD07 skycell.044 2 214.509667604725 52.5181290488877 200 200 null 55009 55011 - - - # If ROWNUM is set to zero, pstamp_request_file will set insert a value for each row beginning with 1 -# If a later input row reuses a value an error occurs +# If a later input row reuses a value an error is preported # ID | ROI Specification | JOB Specification | Images of interest specification Index: /branches/eam_branches/20090820/pswarp/src/pswarpLoop.c =================================================================== --- /branches/eam_branches/20090820/pswarp/src/pswarpLoop.c (revision 25869) +++ /branches/eam_branches/20090820/pswarp/src/pswarpLoop.c (revision 25870) @@ -488,5 +488,8 @@ psString dump_file = psMetadataLookupStr(&status, config->arguments, "DUMP_CONFIG"); if (dump_file) { - pmConfigDump(config, dump_file); + if (!pmConfigDump(config, dump_file)) { + psError(PS_ERR_IO, false, "Unable to dump configuration"); + return false; + } } Index: /branches/eam_branches/20090820/tools/definetargets =================================================================== --- /branches/eam_branches/20090820/tools/definetargets (revision 25870) +++ /branches/eam_branches/20090820/tools/definetargets (revision 25870) @@ -0,0 +1,76 @@ +#!/usr/bin/env perl + +# program to define multiple distribution targets +# use with care + +use warnings; +use strict; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $stage, $dist_group, $filter, $clean, $disable, $save_temps); + +my $dbname; + +GetOptions( + 'stage=s' => \$stage, + 'dist_group=s' => \$dist_group, + 'filter=s' => \$filter, + 'clean' => \$clean, + 'disable' => \$disable, + 'dbname=s' => \$dbname, + 'save-temps' => \$save_temps, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --stage --dist_group --dbname", + -exitval => 3) + unless defined $stage + and defined $dist_group + and defined $dbname; + +my @all_filters = qw( g r i z y ); +my @all_stages = qw( raw chip camera fake warp stack diff ); + +my @filters; +if ($filter) { + die "$filter is not a known filter\n" if ! in_list($filter, \@all_filters); + push @filters, $filter; +} else { + @filters = @all_filters; +} + +my @stages; +if ($stage eq "all") { + @stages = @all_stages; +} else { + @stages = ($stage); +} +my @dist_groups = ($dist_group); + +foreach my $this_dist_group (@dist_groups) { + foreach my $stage (@stages) { + foreach $filter (@filters) { + my $command = "disttool -definetarget -stage $stage -dist_group $this_dist_group -filter $filter.00000"; + $command .= " -clean" if $clean; + $command .= " -set_state disabled" if $disable; + $command .= " -dbname $dbname" if $dbname; + + print "$command\n"; + my $code = system $command; + my $status = $code >> 8; + die "$command failed with $code $status\n" if $code; + } + } +} + +sub in_list { + my $val = shift; + my $array = shift; + + foreach my $a (@$array) { + return 1 if $a eq $val; + } + return 0; +} Index: /branches/eam_branches/20090820/tools/gpc1_find_images.pl =================================================================== --- /branches/eam_branches/20090820/tools/gpc1_find_images.pl (revision 25870) +++ /branches/eam_branches/20090820/tools/gpc1_find_images.pl (revision 25870) @@ -0,0 +1,191 @@ +#! /usr/bin/perl -w + +use DBI; +use Getopt::Std; +getopts('hFARCPp:BMDWvx:n:c:w:O:S:',\%opt); +#getopt('xncwOS:hARCW',\%opt); +unless (exists($opt{O})) { + $opt{O} = ''; +} +unless (exists($opt{S})) { + $opt{S} = ''; +} +if (exists($opt{A})) { + $opt{A} = 1; +} +else { + $opt{A} = 0; +} +if (exists($opt{M})) { + $opt{M} = 1; +} +else { + $opt{M} = 0; +} +if (exists($opt{D})) { + $opt{D} = 1; +} +else { + $opt{D} = 0; +} + +unless (exists($opt{p})) { + $opt{p} = 0; +} + +if (exists($opt{h})) { + print "USAGE find_images.pl ... \n"; + print " -h This help\n"; + print " -R Return raw data.\n"; + print " -C Return chip stage data.\n"; + print " -W Return warp stage data.\n"; + print " -A Return not just fits images.\n"; + print " -F Convert to real filenames.\n"; + print " -P Return previous image to this.\n"; + print " -B Return burntool table.\n"; + print " -M Include masks in results.\n"; + print " -D Return only pre-destreaked images.\n"; + print " -v Be verbose on output.\n"; + print " \n"; + print " -x Specify a exp_id.\n"; + print " -c ...\n"; + print " -w ...\n"; + print " -n ...\n"; + print " \n"; + print " -O Return a specific chip.\n"; + print " -S Return a specific skycell.\n"; + exit(0); +} + + +# Set up the database +use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; +$dbname = 'gpc1'; +$dbserver = 'ippdb01'; +$dbuser = 'ipp'; +$dbpass = 'ipp'; +$db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + +$sth_base = "SELECT warp_id,warpRun.workdir,chipRun.chip_id,chipRun.workdir,rawExp.exp_name,rawExp.exp_id,rawExp.workdir,rawExp.dateobs,rawExp.pon_time from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING (exp_id)"; +# What do we have? +@sth_adds = (); + +if (exists($opt{x})) { + push @sth_adds, "rawExp.exp_id = $opt{x}"; +} +if (exists($opt{n})) { + push @sth_adds, "rawExp.exp_name = '$opt{n}'"; +} +if (exists($opt{c})) { + push @sth_adds, "chipRun.chip_id = $opt{c}"; +} +if (exists($opt{w})) { + push @sth_adds, "warpRun.warp_id = $opt{w}"; +} +if ($#sth_adds != -1) { + $sth_add = join ' AND ', @sth_adds; + $sth = $sth_base . " WHERE " . $sth_add; +} +else { + die "Probable mistake."; +} +#print STDERR "### $sth\n"; +$dr = $db->selectall_arrayref( $sth ); + +foreach $rr (@{ $dr }) { + ($warp_id, $warp_workdir, $chip_id, $chip_workdir, + $exp_name, $exp_id, $exp_workdir, $exp_dateobs, $exp_pontime) = @{ $rr }; +# print (join "\t", @{ $rr }) . "\n"; + $exp_pontime = 0; # To stop warnings, but I want this around. + if (exists($opt{P})) { + $sth_previous = "SELECT exp_name from rawExp where dateobs < '$exp_dateobs' order by dateobs desc limit 1"; + $prev_dr = $db->selectall_arrayref( $sth_previous ); + $prev_exp_name = ${ ${ $prev_dr }[0] }[0]; + $previous_cmd = "$0 -n $prev_exp_name"; + if (exists($opt{R})) { $previous_cmd .= ' -R '; } + if (exists($opt{C})) { $previous_cmd .= ' -C '; } + if (exists($opt{W})) { $previous_cmd .= ' -W '; } + if (exists($opt{B})) { $previous_cmd .= ' -B '; } + if (exists($opt{v})) { $previous_cmd .= ' -v '; } + if ($opt{A}) { $previous_cmd .= ' -A '; } + if ($opt{M}) { $previous_cmd .= ' -M '; } + if ($opt{D}) { $previous_cmd .= ' -D '; } + if ($opt{O} ne '') { $previous_cmd .= " -O $opt{O} "; } + if ($opt{S} ne '') { $previous_cmd .= " -S $opt{S} "; } + if ($opt{p} > 1) { + $opt{p} -= 1; + $previous_cmd .= " -P -p $opt{p} "; + } + if (exists($opt{v})) { + print STDERR "# $previous_cmd\n"; + } + system($previous_cmd); +# exit(0); + } + + + + @scan = (); + if (exists($opt{R})||exists($opt{B})) { + open(NLS,"neb-ls $exp_workdir/${exp_name}/ |") || die "can't neb-ls"; + while () { + chomp; + if (($_ =~ /${opt{O}}/)&&((($opt{A})) || ($_ =~ /fits/))) { + if (exists($opt{B})) { + $_ =~ s/.fits$/.burn.tbl/; + } + push @scan, $_; + } + } + close(NLS); + } + if (exists($opt{C})) { + open(NLS,"neb-ls $chip_workdir/${exp_name}.${exp_id}/ |") || die "can't neb-ls"; + while () { + chomp; + if (($_ =~ /${exp_id}.ch/)&&($_ =~ /$chip_id/)&&($_ =~ /$opt{O}/)&& + ((($opt{A})) || (($_ =~ /fits/) && + ($opt{M} || ($_ !~ /mk.fits/)) && + ($_ !~ /wt.fits/)))) { + if ($opt{D} && ($_ !~ /SR/)) { + next; + } + push @scan, $_; + + } + } + close(NLS); + } + if (exists($opt{W})) { + open(NLS,"neb-ls $warp_workdir/${exp_name}.${exp_id}/ |") || die "can't neb-ls"; + while () { + chomp; +# print "$_\n"; + if (($_ =~ /wrp.${warp_id}.$opt{S}/) && ((($opt{A})) || (($_ =~ /fits/) && + ($opt{M} || ($_ !~ /mask/)) && + ($_ !~ /wt/)))) { + if ($opt{D} && ($_ !~ /SR/)) { + next; + } + push @scan, $_; + } + } + } + %uniq = (); + foreach $f (@scan) { + unless (exists($uniq{$f})) { + $uniq{$f} = 1; + if (exists($opt{F})) { + chomp($f = `neb-locate --path $f`); + } + print "$f\n"; + if (exists($opt{v})) { + print STDERR "$f\n"; + } + } + } +} Index: /branches/eam_branches/20090820/tools/ipp_apply_burntool.pl =================================================================== --- /branches/eam_branches/20090820/tools/ipp_apply_burntool.pl (revision 25869) +++ /branches/eam_branches/20090820/tools/ipp_apply_burntool.pl (revision 25870) @@ -53,4 +53,5 @@ my $burntool = can_run('burntool') or (warn "Can't find burntool" and $missing_tools = 1); my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +my $nebXattr = can_run('neb-xattr') or (warn "Can't find neb-xattr" and $missing_tools = 1); #my $fpack = can_run('fpack') or (warn "Can't find fpack" and $missing_tools = 1); if ($missing_tools) { @@ -190,4 +191,5 @@ } + $status = vsystem ("$neb-xattr --write $outTable user.copies:2"); $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState", $REALRUN); if ($status) { Index: /branches/eam_branches/20090820/tools/neb-ds9.pl =================================================================== --- /branches/eam_branches/20090820/tools/neb-ds9.pl (revision 25870) +++ /branches/eam_branches/20090820/tools/neb-ds9.pl (revision 25870) @@ -0,0 +1,23 @@ +#! /usr/bin/perl -w + +use Getopt::Std; +getopts('AM',\%opt); + +$cmd = 'ds9 -scale mode zscale '; + +unless (exists($opt{A})) { + $cmd .= " -wcs align yes "; +} + +foreach $neb (@ARGV) { + chomp($real_file = `neb-locate --path $neb`); + unless ($real_file =~ /no instances found/) { + if (($real_file =~ /ota/)||(exists($opt{M}))) { + $cmd .= "-medatacube $real_file "; + } + else { + $cmd .= " $real_file "; + } + } +} +system("$cmd"); Index: /branches/eam_branches/20090820/tools/warp_outputs.pl =================================================================== --- /branches/eam_branches/20090820/tools/warp_outputs.pl (revision 25869) +++ /branches/eam_branches/20090820/tools/warp_outputs.pl (revision 25870) @@ -16,4 +16,5 @@ 'VARIANCE' => '.wt.fits', # Variance 'SOURCES' => '.cmf', # Sources + 'PSF' => '.psf', # Point-Spread Function };