Index: branches/eam_branches/20090715/DataStore/Build.PL =================================================================== --- branches/eam_branches/20090715/DataStore/Build.PL (revision 25623) +++ branches/eam_branches/20090715/DataStore/Build.PL (revision 25624) @@ -33,4 +33,5 @@ script_files => [qw( scripts/dsget + scripts/dsgetfileset scripts/dsproductls scripts/dsleech Index: branches/eam_branches/20090715/DataStore/scripts/dsgetfileset =================================================================== --- branches/eam_branches/20090715/DataStore/scripts/dsgetfileset (revision 25624) +++ branches/eam_branches/20090715/DataStore/scripts/dsgetfileset (revision 25624) @@ -0,0 +1,157 @@ +#!/usr/bin/env perl + +# Copyright (C) 2006-2009 Institute for Astromony University of Hawaii +# +# dsgetfileset + +use strict; +use warnings FATAL => qw( all ); + +use vars qw( $VERSION ); +$VERSION = '0.02'; + +use DataStore; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use File::Basename qw( basename ); + +my ($uri, $outdir, $timeout, $verbose); + +GetOptions( + 'uri|u=s' => \$uri, + 'outdir|o=s' => \$outdir, + 'timeout|t=s' => \$timeout, + 'verbose|v' => \$verbose, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --uri --outdir", + -exitval => 3, +) unless defined $uri and defined $outdir; + +# default http request timeout is 30s +$timeout ||= 30; + +my $response = DataStore::FileSet->new( uri => $uri )->request( + ua_args => { timeout => $timeout }, + ); + +unless (defined $response and $response->is_success) { + warn "request failed: ", $response->status_line; + exit($response->code - 300); +} + +# file retreival succeed + +my $data = $response->data; + +unless ($data) { + warn "no files found"; + exit(0); +} + +print "# uri fileid bytes md5sum type \n"; +foreach my $fs (@$data) { + if ($verbose) { + print + $fs->uri, " ", + $fs->fileid, " ", + $fs->bytes, " ", + $fs->md5sum || 0, " ", + $fs->type; + print " ", join(" ", @{$fs->extra}) if defined $fs->extra; + print "\n"; + } + + my $uri = $fs->uri; + my $base = basename($uri); + my $outfile = "$outdir/$base"; + my $md5 = $fs->md5sum; + my $bytes = $fs->bytes; + + my $command = "dsget --uri $uri --filename $outfile --md5 $md5 --bytes $bytes --timeout $timeout"; + print "$command\n" if $verbose; + my $rc = system $command; + if ($rc) { + my $status = $rc >> 8; + print STDERR "dsget failed: rc: $rc status: $status\n"; + exit $status; + } +} + +__END__ + +=pod + +=head1 NAME + +dsgetfilset - download all the Files under a FileSet from a DataStore server + +=head1 SYNOPSIS + + dsgetfileset --uri --outdir + +=head1 DESCRIPTION + +Download the contents of the Data Store Fileset at the supplied uri to a local directory. + +=head1 OPTIONS + +=over 4 + +=item * --uri|-u + +The URI of the file to be retrieved. + +=item * --outdir|-o + +The output destination directory. + +=item * --timeout|-t + +The amount of time (in seconds) to wait for a response from the DataStore +after making an HTTP request. The default is 30s. + +Optional. + +=back + +=head1 CREDITS + +Based on the DataStore modules and scripts by Joshua Hoblitt + +=head1 SUPPORT + +None. + +=head1 AUTHOR + +The IPP Team + +=head1 COPYRIGHT + +Copyright (C) 2006-2009 Institute for Astromony University of Hawaii + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place - Suite 330, Boston, MA 02111-1307, USA. + +The full text of the license can be found in the L Pod as supplied +with Perl 5.8.1 and later. + +=head1 SEE ALSO + +L, L, L, L, L, L + +=cut Index: branches/eam_branches/20090715/DataStore/scripts/dsproductls =================================================================== --- branches/eam_branches/20090715/DataStore/scripts/dsproductls (revision 25623) +++ branches/eam_branches/20090715/DataStore/scripts/dsproductls (revision 25624) @@ -59,5 +59,12 @@ print "# uri fileset datetime type\n"; +my $num = 0; foreach my $fs (@$data) { + $num++; + if (!defined $fs->uri) { + # Somewhow we get here occasaionally I don't know why yet + print STDERR "uri is not defined! skipping fs: $num\n"; + next; + } print $fs->uri, " ", $fs->fileset, " ", $fs->datetime, " ", $fs->type; if ($extra) { Index: branches/eam_branches/20090715/Ohana/src/opihi/cmd.data/rotate.c =================================================================== --- branches/eam_branches/20090715/Ohana/src/opihi/cmd.data/rotate.c (revision 25623) +++ branches/eam_branches/20090715/Ohana/src/opihi/cmd.data/rotate.c (revision 25624) @@ -287,5 +287,5 @@ if (Y >= NY - 1) continue; - c = &in_buff[X * NX*Y]; + c = &in_buff[X + NX*Y]; fx = x - X; fy = y - Y; Index: branches/eam_branches/20090715/Ohana/src/opihi/lib.shell/command.c =================================================================== --- branches/eam_branches/20090715/Ohana/src/opihi/lib.shell/command.c (revision 25623) +++ branches/eam_branches/20090715/Ohana/src/opihi/lib.shell/command.c (revision 25624) @@ -5,6 +5,9 @@ int i, status, argc; - char **argv, **targv; + char **argv, **targv, *rawline; Command *cmd; + + rawline = NULL; + // rawline = strcreate (line); /* force a space between ! and first word: !ls becomes ! ls */ @@ -31,5 +34,8 @@ argv = parse_commands (line, &argc); - if (argc == 0) return (TRUE); /* empty command or assignment */ + if (argc == 0) { + FREE (rawline); + return (TRUE); /* empty command or assignment */ + } /* save the original values of argv since command may modify the array */ @@ -54,4 +60,5 @@ msg = get_variable_ptr ("ERRORMSG"); if (msg != (char *) NULL) gprint (GP_ERR, "%s\n", msg); + // gprint (GP_ERR, "error on line: %s\n", rawline); } @@ -62,4 +69,5 @@ # endif + FREE (rawline); return (status); } Index: branches/eam_branches/20090715/dbconfig/changes.txt =================================================================== --- branches/eam_branches/20090715/dbconfig/changes.txt (revision 25623) +++ branches/eam_branches/20090715/dbconfig/changes.txt (revision 25624) @@ -1082,5 +1082,5 @@ ALTER TABLE rcRun ADD COLUMN fault SMALLINT DEFAULT 0; -ALTER TABLE rcRun ADD COLUMN status_fs VARCHAR(64) AFTER state; +ALTER TABLE rcRun ADD COLUMN status_fs_name VARCHAR(64) AFTER state; ALTER TABLE diffSkyfile ADD COLUMN deconv_max FLOAT AFTER kernel_yy; @@ -1248,2 +1248,52 @@ PRIMARY KEY(label) ) ENGINE=innodb DEFAULT CHARSET=latin1; + + +ALTER TABLE pzDownloadImfile ADD COLUMN bytes INT; +ALTER TABLE pzDownloadImfile ADD COLUMN md5sum VARCHAR(32); +ALTER TABLE newImfile ADD COLUMN bytes INT; +ALTER TABLE newImfile ADD COLUMN md5sum VARCHAR(32); +ALTER TABLE rawImfile ADD COLUMN bytes INT; +ALTER TABLE rawImfile ADD COLUMN md5sum VARCHAR(32); +ALTER TABLE rawImfile ADD COLUMN burntool_state SMALLINT; + +-- Vesion 1.1.56 + +-- Merge the rcDSProduct and rcDestination tables + +ALTER TABLE rcDestination ADD COLUMN dbname varchar(64) AFTER last_fileset; +ALTER TABLE rcDestination ADD COLUMN dbhost varchar(64) AFTER dbname; + +UPDATE rcDestination JOIN rcDSProduct USING(prod_id) + SET rcDestination.name = rcDSProduct.name, + rcDestination.dbname = rcDSProduct.dbname, + rcDestination.dbhost = rcDSProduct.dbhost; + +ALTER TABLE rcDestination DROP foreign key rcDestination_ibfk_1 ; +ALTER TABLE rcDestination DROP key prod_id; +ALTER TABLE rcDestination DROP COLUMN prod_id; + +-- change references to rcDSProduct in rcDSFileset to reference rcDestination + +ALTER TABLE rcDSFileset DROP foreign key rcDSFileset_ibfk_2; +ALTER TABLE rcDSFileset DROP key prod_id; +ALTER TABLE rcDSFileset CHANGE COLUMN prod_id dest_id BIGINT NOT NULL DEFAULT '0'; +ALTER TABLE rcDSFileset ADD KEY (dest_id); +ALTER TABLE rcDSFileset ADD FOREIGN KEY (dest_id) REFERENCES rcDestination(dest_id); + +-- finally drop the obsolete table +DROP TABLE rcDSProduct; + +-- The following changes should have been made to the gpc1 database ages ago +-- do not apply if these columns exist +ALTER TABLE pstampRequest DROP COLUMN outFileset; +ALTER TABLE pstampRequest ADD COLUMN name VARCHAR(64) UNIQUE AFTER state; +ALTER TABLE pstampRequest ADD COLUMN reqType VARCHAR(16) AFTER name; +ALTER TABLE pstampRequest ADD COLUMN outProduct VARCHAR(64) after reqType; +ALTER TABLE pstampRequest ADD COLUMN fault SMALLINT AFTER uri; +-- end of old changes + +-- This change is new +ALTER TABLE pstampRequest ADD COLUMN label VARCHAR(64) AFTER reqType; + +ALTER TABLE distRun ADD COLUMN magic_ds_id BIGINT AFTER stage_id; Index: branches/eam_branches/20090715/dbconfig/config.md =================================================================== --- branches/eam_branches/20090715/dbconfig/config.md (revision 25623) +++ branches/eam_branches/20090715/dbconfig/config.md (revision 25624) @@ -2,4 +2,4 @@ pkg_name STR ippdb pkg_namespace STR ippdb - pkg_version STR 1.1.54 + pkg_version STR 1.1.55 END Index: branches/eam_branches/20090715/dbconfig/dist.md =================================================================== --- branches/eam_branches/20090715/dbconfig/dist.md (revision 25623) +++ branches/eam_branches/20090715/dbconfig/dist.md (revision 25624) @@ -14,4 +14,5 @@ stage STR 64 stage_id S64 0 + magic_ds_id S64 0 label STR 64 # Key outroot STR 255 Index: branches/eam_branches/20090715/dbconfig/new.md =================================================================== --- branches/eam_branches/20090715/dbconfig/new.md (revision 25623) +++ branches/eam_branches/20090715/dbconfig/new.md (revision 25624) @@ -22,4 +22,6 @@ uri STR 255 epoch UTC 0001-01-01T00:00:00Z + bytes S32 0 + md5sum STR 32 END Index: branches/eam_branches/20090715/dbconfig/pstamp.md =================================================================== --- branches/eam_branches/20090715/dbconfig/pstamp.md (revision 25623) +++ branches/eam_branches/20090715/dbconfig/pstamp.md (revision 25624) @@ -24,4 +24,5 @@ name STR 64 # UNIQUE reqType STR 16 + label STR 64 outProduct STR 64 uri STR 255 Index: branches/eam_branches/20090715/dbconfig/raw.md =================================================================== --- branches/eam_branches/20090715/dbconfig/raw.md (revision 25623) +++ branches/eam_branches/20090715/dbconfig/raw.md (revision 25624) @@ -136,3 +136,6 @@ epoch UTC 0001-01-01T00:00:00Z magicked S64 0 + bytes S32 0 + md5sum STR 32 + burntool_state S16 0 END Index: branches/eam_branches/20090715/dbconfig/rc.md =================================================================== --- branches/eam_branches/20090715/dbconfig/rc.md (revision 25623) +++ branches/eam_branches/20090715/dbconfig/rc.md (revision 25624) @@ -1,7 +1,11 @@ -rcDSProduct METADATA - prod_id S64 0 # Primary Key - name STR 64 - dbname STR 64 - dbhost STR 64 +rcDestination METADATA + dest_id S64 0 # Primary Key + name STR 64 + status_uri STR 255 + comment STR 255 + last_fileset STR 255 + dbname STR 64 + dbhost STR 64 + state STR 64 # Key END @@ -9,5 +13,5 @@ fs_id S64 0 # Primary Key dist_id S64 0 # fkey(dist_id) ref distRun(dist_id) - prod_id S64 0 # fkey(prod_id) ref rcDSProduct(prod_id) + dest_id S64 0 # fkey(dest_id) ref rcDestination(dest_id) name STR 255 state STR 64 @@ -15,13 +19,4 @@ END -rcDestination METADATA - dest_id S64 0 # Primary Key - prod_id S64 0 # fkey(prod_id) ref rcDSProduct(prod_id) - name STR 64 - status_uri STR 255 - comment STR 255 - last_fileset STR 255 - state STR 64 # Key -END rcInterest METADATA Index: branches/eam_branches/20090715/dbconfig/summitcopy.md =================================================================== --- branches/eam_branches/20090715/dbconfig/summitcopy.md (revision 25623) +++ branches/eam_branches/20090715/dbconfig/summitcopy.md (revision 25624) @@ -57,4 +57,6 @@ epoch UTC 0001-01-01T00:00:00Z hostname STR 64 + bytes S32 0 + md5sum STR 32 END Index: branches/eam_branches/20090715/extsrc/gpcsw/Make.Common.fixed =================================================================== --- branches/eam_branches/20090715/extsrc/gpcsw/Make.Common.fixed (revision 25623) +++ branches/eam_branches/20090715/extsrc/gpcsw/Make.Common.fixed (revision 25624) @@ -670,6 +670,6 @@ $(DIR_BIN)/%: $(DIR_BIN)/%-$(VERSION) @echo "--> Installing link to $(notdir $<) as $(notdir $@)" - @-mv $@ $@.$(shell date +%H%M%S).TEXT_FILE_BUSY 2> /dev/null || /bin/true - @-rm -f $(DIR_BIN)/*.TEXT_FILE_BUSY 2> /dev/null || /bin/true + @-mv $@ $@.$(shell date +%H%M%S).TEXT_FILE_BUSY 2> /dev/null || true + @-rm -f $(DIR_BIN)/*.TEXT_FILE_BUSY 2> /dev/null || true @ln $< $@ # @-setuidinst $@ @@ -677,10 +677,10 @@ $(DIR_BIN)/%-$(VERSION): scripts/%.sh @echo "--> Installing sh script $(notdir $@)" - @-mv $@ $@.$(shell date +%H%M%S).TEXT_FILE_BUSY 2> /dev/null || /bin/true + @-mv $@ $@.$(shell date +%H%M%S).TEXT_FILE_BUSY 2> /dev/null || true @$(INSTALL) -m 0555 $< $@ || ( rm -f $@ && exit 1 ) $(DIR_BIN)/%-$(VERSION): $(EXECNAME) @echo "--> Installing program $(notdir $@)" - @-mv $@ $@.$(shell date +%H%M%S).TEXT_FILE_BUSY 2> /dev/null || /bin/true + @-mv $@ $@.$(shell date +%H%M%S).TEXT_FILE_BUSY 2> /dev/null || true @$(INSTSTRIP) -m 0755 $< $@ || ( rm -f $@ && exit 1 ) @chmod 0555 $@ Index: branches/eam_branches/20090715/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c =================================================================== --- branches/eam_branches/20090715/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c (revision 25623) +++ branches/eam_branches/20090715/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c (revision 25624) @@ -300,6 +300,5 @@ } - - /* Set the read noise for the noise gate that cells must pass */ +/* Set the read noise for the noise gate that cells must pass */ } else if(strncmp(argv[i], "RN=", 3) == 0) { /* EADU=e */ if(sscanf(argv[i]+3, "%d", &MAX_READ_NOISE) != 1) { @@ -601,10 +600,11 @@ /* Fix up the streaks (don't bother if table only) */ - if(!tableonly) { +// Why the heck do I get to skip this if table only? Makes no sense! +// if(!tableonly) { persist_fix(naxis1-ovrscan1, naxis2-ovrscan2, naxis1, buf, OTA+cell); /* Tell us about it? */ if(VERBOSE & VERB_NORM) persist_blab(OTA+cell); - } +// } } Index: branches/eam_branches/20090715/extsrc/gpcsw/gpcsrc/fits/burntool/psfstamp.c =================================================================== --- branches/eam_branches/20090715/extsrc/gpcsw/gpcsrc/fits/burntool/psfstamp.c (revision 25623) +++ branches/eam_branches/20090715/extsrc/gpcsw/gpcsrc/fits/burntool/psfstamp.c (revision 25624) @@ -256,5 +256,5 @@ int k, l, cellx, celly, ota_xid, ota_yid, nfwave, nqavg; double xota, yota, xfp, yfp, phi; - double fwhm[3], q[3], fw[MAXPSFMEDIAN]; + double fwhm[3], q[5], fw[MAXPSFMEDIAN]; double m2[MAXPSFMEDIAN], qp[MAXPSFMEDIAN], qc[MAXPSFMEDIAN]; double qt[MAXPSFMEDIAN], fwavg[MAXPSFMEDIAN]; Index: branches/eam_branches/20090715/ippScripts/scripts/chip_imfile.pl =================================================================== --- branches/eam_branches/20090715/ippScripts/scripts/chip_imfile.pl (revision 25623) +++ branches/eam_branches/20090715/ippScripts/scripts/chip_imfile.pl (revision 25624) @@ -19,4 +19,5 @@ use IPC::Cmd 0.36 qw( can_run run ); use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); use PS::IPP::Config 1.01 qw( :standard ); use File::Temp qw( tempfile ); @@ -143,7 +144,142 @@ ## XXX make the feature more general? my $useDeburnedImage = metadataLookupBool($recipeData, 'USE.DEBURNED.IMAGE'); - if ($useDeburnedImage && $deburned) { - $uri =~ s/fits$/burn.fits/; - &my_die("Couldn't find deburned input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri); + + if ($useDeburnedImage) { + my $useBestBurntool = metadataLookupBool($recipeData, 'USE.BEST.BURNTOOL'); + + ## Check that we have required programs: +# print STDERR "Inside burntool loop!\n"; + my $regtool = can_run('regtool') or &my_die ("Can't find regtool", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my $burntool = can_run('burntool') or &my_die ("Can't find burntool", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + + ## Check the current burntool processing version: + my $regtool_state_cmd = "$regtool -processedimfile -exp_id $exp_id -class_id $class_id -limit 1"; + if (defined($dbname)) { + $regtool_state_cmd .= " -dbname $dbname"; + } + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $regtool_state_cmd, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform regtool: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + my $regData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse regtool metadata", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my $regData2 = parse_md_list($regData); + + my $burntoolState = 999; + foreach my $regEntry (@$regData2) { +# print "$regEntry->{exp_id} $regEntry->{burntool_state}\n"; + + if ($regEntry->{exp_id} == $exp_id) { + $burntoolState = $regEntry->{burntool_state}; + } + } + if ($burntoolState == 999) { + &my_die("Unable to find burntool_state in metadata", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + if ($burntoolState == 32767) { + $burntoolState = 0; + } + + ## Read camera config to get the current good burntool state : + ## XXX This is extremely slow. Any better way to do this? + my $camera_config_cmd = "$ppConfigDump -camera $camera -dump-camera -"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $camera_config_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + my $camData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse ppConfigDump metadata", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + + my $burntoolStateGood; + foreach my $camEntry (@$camData) { + if ($camEntry->{name} eq "BURNTOOL.STATE.GOOD") { + $burntoolStateGood = $camEntry->{value}; + } + } + + if (abs($burntoolState) != $burntoolStateGood) { + if ($useBestBurntool) { + &my_die("Image burntool version does not match current accepted version.", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + else { + warn ("Image burntool version does not match current accepted version. Continuing as requested."); + # Internally pretend that we're at a good version. + if ($burntoolState < -10) { + $burntoolState = -1 * $burntoolStateGood; + } + elsif ($burntoolState > 10) { + $burntoolState = $burntoolStateGood; + } + elsif ($burntoolState == 0) { + # You've told me to use a deburned image, and that you don't care if it's the most recent. The database has told me + # that burntool has never been run on this image. We'll hope the database is wrong, but I don't really trust that. + warn ("burntool_state suggests no table will be found. This will likely crash."); + $burntoolState = -1 * $burntoolStateGood; + } + else { + &my_die("No valid burntool table will be found.", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + } + } + + ## We now know that we have an image that has been burntooled. + my ($tempFile, $tempName) = tempfile( "/tmp/chip.$exp_id.$class_id.deburned.XXXX", + UNLINK => !$save_temps, SUFFIX => '.fits' ); + + # get the UNIX version of the (possible) neb: or path: filename + my $uriReal = $ipprc->file_resolve( $uri ); + + # funpack into the temp file. + my $funpack_cmd = "$funpack -S $uriReal > $tempName"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + ## Construct commands to apply the pre-calculated burntool trailfits to the pixel data. + my ($burntoolTable_uri, $burntoolTable_uriReal); + my $burntool_cmd = "$burntool "; + + if ($burntoolState == -1 * $burntoolStateGood) { # Burntool information stored in an external table. + $burntoolTable_uri = $uri; + $burntoolTable_uri =~ s/fits$/burn.tbl/; + $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri ); + unless ($ipprc->file_exists($burntoolTable_uri)) { + &my_die("Couldn't find burntool table: $burntoolTable_uri",$exp_id,$chip_id,$class_id, $PS_EXIT_SYS_ERROR); + } + + $burntool_cmd .= "$tempName in=${burntoolTable_uriReal} apply=t"; + } + elsif ($burntoolState == $burntoolStateGood) { # Burntool information stored in a header table. + $burntool_cmd .= "$tempName apply=t"; + } + else { + &my_die("Image data not properly burntooled, impossible state", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + ## Run burntool to change the pixels of tempfile, and repoint $uri to that file. + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = + run(command => $burntool_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform burntool: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + $uri = $tempName; + unless ($ipprc->file_exists($uri)) { + &my_die("Couldn't find deburned input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + +# print STDERR "$uri $uriReal $tempName $burntoolTable_uri $burntoolTable_uriReal $burntool_cmd $save_temps\n"; +# exit(100); } @@ -152,48 +288,48 @@ if ($tiltystreakApply) { - my $tiltystreakByClass = metadataLookupMD($recipeData, 'TILTYSTREAK.BY.CLASS'); - my $tiltystreakClassApply = metadataLookupBool($tiltystreakByClass, "APPLY.$class_id"); - if ($tiltystreakClassApply) { - - my $tiltystreakClassOptions = metadataLookupStr($tiltystreakByClass, "OPTIONS.$class_id"); - if ($tiltystreakClassOptions eq "none") { - $tiltystreakClassOptions = ""; - } - - my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf); - - # XXX make these optional (must be built by psbuild as well...) - my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); - my $tiltystreak = can_run('tiltystreak') or &my_die ("Can't find tiltystreak", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); - - # create an temporary output file: - my ($tempFile, $tempName) = tempfile( "/tmp/chip.$exp_id.$class_id.tmp.XXXX", UNLINK => !$save_temps ); - - print "uri: $uri\n"; - - # get the UNIX version of the (possible) neb: or path: filename - my $uriReal = $ipprc->file_resolve( $uri ); - - print "uriReal: $uriReal\n"; - - # unpack the data (is a NOP if not compressed) - $command = "$funpack -S $uriReal > $tempName"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); - } - - # run tiltystreak - $command = "$tiltystreak $tempName $tiltystreakClassOptions"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform tiltystreak: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); - } - - # supply the output file as the new input file - $uri = $tempName; - } + my $tiltystreakByClass = metadataLookupMD($recipeData, 'TILTYSTREAK.BY.CLASS'); + my $tiltystreakClassApply = metadataLookupBool($tiltystreakByClass, "APPLY.$class_id"); + if ($tiltystreakClassApply) { + + my $tiltystreakClassOptions = metadataLookupStr($tiltystreakByClass, "OPTIONS.$class_id"); + if ($tiltystreakClassOptions eq "none") { + $tiltystreakClassOptions = ""; + } + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf); + + # XXX make these optional (must be built by psbuild as well...) + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my $tiltystreak = can_run('tiltystreak') or &my_die ("Can't find tiltystreak", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + + # create an temporary output file: + my ($tempFile, $tempName) = tempfile( "/tmp/chip.$exp_id.$class_id.tmp.XXXX", UNLINK => !$save_temps ); + + print "uri: $uri\n"; + + # get the UNIX version of the (possible) neb: or path: filename + my $uriReal = $ipprc->file_resolve( $uri ); + + print "uriReal: $uriReal\n"; + + # unpack the data (is a NOP if not compressed) + $command = "$funpack -S $uriReal > $tempName"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + # run tiltystreak + $command = "$tiltystreak $tempName $tiltystreakClassOptions"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform tiltystreak: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + # supply the output file as the new input file + $uri = $tempName; + } } @@ -253,21 +389,5 @@ } chomp $cmdflags; - if (($camera ne "GPC1") or ($class_id ne "XY27") or ($exp_id < 53171)) { - ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag - } else { - # hack to kick ota 27 out of subsequent analysis by setting quality flag to a non-zero value - my ($before, $after) = split " -quality ", $cmdflags; - - # get the current value - ($quality) = $after =~ /^(\d+)/; - $after = substr($after, length($quality)); - - # replace it if it is zero - $quality = 42 if !$quality; - - # rebuild the cmdflags - $cmdflags = $before . " -quality $quality"; - $cmdflags .= " $after" if $after; - } + ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag } Index: branches/eam_branches/20090715/ippScripts/scripts/dist_make_fileset.pl =================================================================== --- branches/eam_branches/20090715/ippScripts/scripts/dist_make_fileset.pl (revision 25623) +++ branches/eam_branches/20090715/ippScripts/scripts/dist_make_fileset.pl (revision 25624) @@ -39,5 +39,5 @@ # Parse the command-line arguments -my ($dist_id, $dist_dir, $target_id, $stage, $stage_id, $prod_id, $product_name, $ds_dbhost, $ds_dbname); +my ($dist_id, $dist_dir, $target_id, $stage, $stage_id, $dest_id, $product_name, $ds_dbhost, $ds_dbname); my ($dbname, $save_temps, $verbose, $no_update, $logfile); @@ -48,5 +48,5 @@ 'stage=s' => \$stage, # raw, chip, camera, fake, warp, stack, or diff 'stage_id=s' => \$stage_id, # exp_id, chip_id, etc. - 'prod_id=s' => \$prod_id, # id for the product + 'dest_id=s' => \$dest_id, # id for the product 'product_name=s' => \$product_name, # location of the data store directory for this product 'ds_dbhost=s' => \$ds_dbhost, # database host for the datastore database @@ -60,5 +60,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --dist_id --dist_dir --target_id --stage --stage_id --prod_id --ds_dbhost --ds_dbname", +pod2usage( -msg => "Required options: --dist_id --dist_dir --target_id --stage --stage_id --dest_id --ds_dbhost --ds_dbname", -exitval => 3) unless defined $dist_id and @@ -67,5 +67,5 @@ defined $stage and defined $stage_id and - defined $prod_id and + defined $dest_id and defined $product_name and defined $ds_dbhost and @@ -79,7 +79,8 @@ my $fs_tag = get_fileset_tag($ipprc, $stage, $stage_id, $dbname); -&my_die("failed to lookup fileset tag", $dist_id, $prod_id, $PS_EXIT_UNKNOWN_ERROR) if !$fs_tag; - -my $fileset_name = "$fs_tag.$stage.$stage_id.$dist_id.$prod_id"; +&my_die("failed to lookup fileset tag", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR) if !defined $fs_tag; + +my $fileset_name = $fs_tag ? "$fs_tag." : ""; +$fileset_name .= "$stage.$stage_id.$dist_id.$dest_id"; print "$fileset_name\n"; @@ -89,5 +90,5 @@ my $dbinfo_file = "dbinfo.$stage.$stage_id.mdc"; if (! -e "$dist_dir/$dbinfo_file" ) { - &my_die("dbinfo file for dist run $dbinfo_file not found", $dist_id, $prod_id, $PS_EXIT_UNKNOWN_ERROR); + &my_die("dbinfo file for dist run $dbinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR); } print "dbinfo file $dbinfo_file exists\n" if $verbose; @@ -96,5 +97,5 @@ my $dirinfo_file = "dirinfo.$stage.$stage_id.mdc"; if (! -e "$dist_dir/$dirinfo_file" ) { - &my_die("dirinfo file for dist run $dirinfo_file not found", $dist_id, $prod_id, $PS_EXIT_UNKNOWN_ERROR); + &my_die("dirinfo file for dist run $dirinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR); } print "dirinfo file $dirinfo_file exists\n" if $verbose; @@ -116,9 +117,9 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform $command error_code: $error_code", $dist_id, $prod_id, $error_code); + &my_die("Unable to perform $command error_code: $error_code", $dist_id, $dest_id, $error_code); } my $metadata = $mdcParser->parse (join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $dist_id, $prod_id, $PS_EXIT_PROG_ERROR); + &my_die("Unable to parse metadata config doc", $dist_id, $dest_id, $PS_EXIT_PROG_ERROR); $components = parse_md_list($metadata); @@ -153,5 +154,7 @@ # set the product specific columns in product list - $command .= " --ps0 $target_id --ps1 $stage --ps2 $stage_id --ps3 $fs_tag"; + my $prod_col_3 = $fs_tag ? $fs_tag : "$stage.$stage_id"; + + $command .= " --ps0 $target_id --ps1 $stage --ps2 $stage_id --ps3 $prod_col_3"; $command .= " --dbname $ds_dbname"; # XXX: notyet --dbhost $ds_dbhost @@ -161,10 +164,10 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform $command error_code: $error_code", $dist_id, $prod_id, $error_code); + &my_die("Unable to perform $command error_code: $error_code", $dist_id, $dest_id, $error_code); } } { - my $command = "$disttool -addfileset -dist_id $dist_id -prod_id $prod_id -name $fileset_name"; + my $command = "$disttool -addfileset -dist_id $dist_id -dest_id $dest_id -name $fileset_name"; $command .= " -dbname $dbname" if $dbname; @@ -175,5 +178,5 @@ # We need to have revertfileset check whether the fileset exists and do dsreg -del if it does $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform $command error_code: $error_code", $dist_id, $prod_id, $error_code); + &my_die("Unable to perform $command error_code: $error_code", $dist_id, $dest_id, $error_code); } } @@ -211,8 +214,6 @@ my $dbname = shift; - if ($stage eq 'stack') { - return "stack.$stage_id"; - } elsif ($stage eq 'diff') { - return "diff.$stage_id"; + if (($stage eq 'stack') or ($stage eq 'diff')) { + return ""; } @@ -248,5 +249,5 @@ my $tag = $ref->{exp_name}; - return $tag; + return "$tag"; } @@ -254,11 +255,11 @@ my $msg = shift; my $dist_id = shift; - my $prod_id = shift; + my $dest_id = shift; my $fault = shift; - # TODO: disttool -adddsfileset -prod_id $prod_id -dist_id $dist_id -fault $fault + # TODO: disttool -adddsfileset -dest_id $dest_id -dist_id $dist_id -fault $fault print STDERR "$msg\n"; - my $command = "$disttool -addfileset -dist_id $dist_id -prod_id $prod_id -fault $fault"; + my $command = "$disttool -addfileset -dist_id $dist_id -dest_id $dest_id -fault $fault"; $command .= " -dbname $dbname" if $dbname; Index: branches/eam_branches/20090715/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- branches/eam_branches/20090715/ippScripts/scripts/ipp_cleanup.pl (revision 25623) +++ branches/eam_branches/20090715/ippScripts/scripts/ipp_cleanup.pl (revision 25624) @@ -123,7 +123,8 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); - if (!$config_file or ! -e $config_file) { +# if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for chipRun $stage_id $class_id " - . " because config file is missing\n"; + . " because config file ($config_file) is missing\n"; $status = 0; } @@ -132,7 +133,7 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping scrubbed for chipRun $stage_id $class_id " - . " because config file is present\n"; + . " because config file ($config_file) is present\n"; $status = 0; } @@ -144,7 +145,7 @@ # delete the temporary image datafiles - addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id); +# addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); +# addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); +# addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id); addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id); addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id); @@ -236,6 +237,6 @@ if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); - - if (!$config_file or ! -e $config_file) { + + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for camRun $stage_id because config file is missing\n"; $status = 0; @@ -245,5 +246,5 @@ my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for camRun $stage_id because config file ($config_file) is present\n"; $status = 0; @@ -329,5 +330,5 @@ my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" . " because config file is missing\n"; @@ -338,5 +339,5 @@ my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping scrubbed for warpRun $stage_id $skycell_id" . " because config file is present\n"; @@ -344,24 +345,27 @@ } } - if ($status) { - # delete the temporary image datafiles - addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id ); - addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); - addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id); + if ($status) { + if ($skyfile->{quality} != 8007) { + my @files = (); + + # delete the temporary image datafiles + addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id ); + addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); + addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id); # addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id); - - if ($mode eq "goto_purged") { - # additional files to remove for 'purge' mode - addFilename(\@files, "PSWARP.BIN1", $path_base, $skycell_id ); - addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id ); - addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id ); - # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base); - - # XXX: do we want to delete these? - # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id); - # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id); - } + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename(\@files, "PSWARP.BIN1", $path_base, $skycell_id ); + addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id ); + addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id ); + # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base); + + # XXX: do we want to delete these? + # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id); + # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id); + } # actual command to delete the files - $status = &delete_files (\@files); + $status = &delete_files (\@files); + } } @@ -433,16 +437,14 @@ my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); - $config_file =~ s%^file://%%; - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for stackRun $stage_id $skycell_id" . " because config file is missing\n"; $status = 0; } - $config_file = 'file://' . $config_file; } elsif ($mode eq "goto_scrubbed") { my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); - $config_file =~ s%^file://%%; - if ($config_file and -e $config_file) { + + if ($ipprc->file_exists($config_file)) { print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" . " because config file is present\n"; @@ -451,4 +453,5 @@ } if ($status) { + my @files = (); # delete the temporary image datafiles addFilename(\@files, "PPSTACK.OUTPUT", $path_base, $skycell_id); @@ -535,16 +538,14 @@ my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); - $config_file =~ s%^file://%%; - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" . " because config file ($config_file) is missing\n"; $status = 0; } - $config_file = 'file://' . $config_file; } elsif ($mode eq "goto_scrubbed") { my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); - $config_file =~ s%^file://%%; - if ($config_file and -e $config_file) { + + if ($ipprc->file_exists($config_file)) { print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" . " because config file ($config_file) is present\n"; @@ -553,4 +554,5 @@ } if ($status) { + my @files = (); # delete the temporary image datafiles addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id); @@ -581,8 +583,8 @@ } - print STDERR "MY FILES: @files\n"; +# print STDERR "MY FILES: @files\n"; $status = &delete_files(\@files); } - print STDERR "MY STATUS: $status\n"; +# print STDERR "MY STATUS: $status\n"; if ($status) { my $command = "$difftool -diff_id $stage_id"; @@ -675,5 +677,5 @@ my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for fakeRun $stage_id $class_id " . " because config file is missing\n"; @@ -684,5 +686,5 @@ my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping scrubbed for fakeRun $stage_id $class_id " . " because config file is present\n"; @@ -795,5 +797,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for detrend.process.imfile $stage_id $class_id " . " because config file is missing\n"; @@ -804,5 +806,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping scrubbed for detrend.process.imfile $stage_id $class_id " . " because config file is present\n"; @@ -942,5 +944,5 @@ my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base); - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; $status = 0; @@ -950,5 +952,5 @@ my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; $status = 0; @@ -1094,4 +1096,5 @@ if ($status) { + my @files = (); # delete the temporary image datafiles # There's no convenient way to get the detrend type, so I'm queueing all of them for deletion. @@ -1227,5 +1230,4 @@ run(command => $command, verbose => $verbose); unless ($success) { - print STDERR "WTF: $success TTT $error_code QQQ\n"; $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); @@ -1316,5 +1318,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; $status = 0; @@ -1324,5 +1326,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; $status = 0; @@ -1450,5 +1452,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; $status = 0; @@ -1458,5 +1460,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; $status = 0; @@ -1591,5 +1593,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for $stage $stage_id $class_id " . " because config file is missing\n"; @@ -1600,5 +1602,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping scrubbed for $stage $stage_id $class_id " . " because config file is present\n"; @@ -1731,5 +1733,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; $status = 0; @@ -1739,5 +1741,5 @@ my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); - if ($config_file and -e $config_file) { + if ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; $status = 0; @@ -1833,26 +1835,36 @@ { my $files = shift; # reference to a list of files to unlink - -# open(TMPLOG,">>/tmp/czw.cleanup.log"); -# flock(TMPLOG,2); +# my $test_verbose = 1; + +# if ($test_verbose == 1) { +# open(TMPLOG,">>/tmp/czw.cleanup.log"); +# flock(TMPLOG,2); +# } # this script is, of course, very dangerous. foreach my $file (@$files) { - print STDERR "unlinking $file\n"; -# print TMPLOG "$stage $stage_id $file"; -# my $ff = $file; -# $ff =~ s%^file://%%; -# unless (-e $ff) { -# print TMPLOG "\t File not found\n"; + print STDERR "unlinking $stage $stage_id $file"; + unless ($ipprc->file_exists($file)) { + print STDERR "\t File not found\n"; + } + else { + print STDERR "\n"; + } + +# if ($test_verbose == 1) { +# print TMPLOG "$stage $stage_id $file"; + +# else { +# print TMPLOG "\n"; +# } # } -# else { -# print TMPLOG "\n"; -# } $ipprc->file_delete($file); } -# flock(TMPLOG,8); -# close(TMPLOG); +# if ($test_verbose == 1) { +# flock(TMPLOG,8); +# close(TMPLOG); +# } return 1; Index: branches/eam_branches/20090715/ippScripts/scripts/magic_destreak_revert.pl =================================================================== --- branches/eam_branches/20090715/ippScripts/scripts/magic_destreak_revert.pl (revision 25623) +++ branches/eam_branches/20090715/ippScripts/scripts/magic_destreak_revert.pl (revision 25624) @@ -191,5 +191,5 @@ } -revert_files($image, $mask, $weight, $sources, $astrom, $bimage, $bmask, $bweight, $bsources, $bastrom); +revert_files($replace, $image, $mask, $weight, $sources, $astrom, $bimage, $bmask, $bweight, $bsources, $bastrom); if ($stage eq "diff") { @@ -203,10 +203,10 @@ $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base); $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base); - revert_files($image, $mask, $weight, $sources, undef, $bimage, $bmask, $bweight, $bsources, undef); + revert_files($replace, $image, $mask, $weight, $sources, undef, $bimage, $bmask, $bweight, $bsources, undef); } # now revert the row in the database { - my $command = "$magicdstool -revertdestreakedfile"; + my $command = "$magicdstool -revertdestreakedfile -i_am_sure"; $command .= " -magic_ds_id $magic_ds_id"; $command .= " -component $component"; @@ -231,4 +231,7 @@ sub revert_files { + my $replace = shift; + return if !$replace; + my $image = shift; my $mask = shift; Index: branches/eam_branches/20090715/ippScripts/scripts/rcserver_checkstatus.pl =================================================================== --- branches/eam_branches/20090715/ippScripts/scripts/rcserver_checkstatus.pl (revision 25623) +++ branches/eam_branches/20090715/ippScripts/scripts/rcserver_checkstatus.pl (revision 25624) @@ -40,10 +40,9 @@ # Parse the command-line arguments -my ($dest_id, $prod_id, $status_uri, $last_fileset, $limit); +my ($dest_id, $status_uri, $last_fileset, $limit); my ($dbname, $save_temps, $verbose, $no_update, $logfile); GetOptions( 'dest_id=s' => \$dest_id, # destination identifier - 'prod_id=s' => \$prod_id, # product identifier 'status_uri=s' => \$status_uri, # uri for the desination's status data store product 'last_fileset=s' => \$last_fileset, # name of last fileset seen on this datastore Index: branches/eam_branches/20090715/ippScripts/scripts/register_imfile.pl =================================================================== --- branches/eam_branches/20090715/ippScripts/scripts/register_imfile.pl (revision 25623) +++ branches/eam_branches/20090715/ippScripts/scripts/register_imfile.pl (revision 25624) @@ -33,6 +33,7 @@ my $ppStats = can_run( 'ppStats' ) or (warn "Can't find ppStats" and $missing_tools = 1); my $ppStatsFromMetadata = can_run( 'ppStatsFromMetadata' ) or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); - -my ($cache, $exp_id, $tmp_class_id, $tmp_exp_name, $uri, $dbname, $verbose, $no_update, $no_op, $logfile); +my $ppConfigDump = can_run( 'ppConfigDump' ) or (warn "Can't find ppConfigDump" and $missing_tools = 1); + +my ($cache, $exp_id, $tmp_class_id, $tmp_exp_name, $uri, $bytes, $md5sum, $dbname, $verbose, $no_update, $no_op, $logfile); GetOptions( 'caches' => \$cache, @@ -41,4 +42,6 @@ 'tmp_exp_name|n=s' => \$tmp_exp_name, 'uri|u=s' => \$uri, + 'bytes=s' => \$bytes, + 'md5sum=s' => \$md5sum, 'dbname|d=s' => \$dbname, # Database name 'verbose' => \$verbose, # Print to stdout @@ -106,4 +109,45 @@ chomp $out2; $cmdflags = $out2; + + # Manually parse the burntool_state entry. + my $burntoolState = 0; + my $isGPC1 = 0; + my $burntoolStateGood = 0; + foreach my $line (split /\n/, $out1) { + if ($line =~ /FPA.BURNTOOL.APPLIED/) { + $line =~ s/^\s+//; + $burntoolState = (split /\s+/, $line)[2]; + } + if ($line =~ /FPA.CAMERA/) { + $line =~ s/^\s+//; + if ((split /\s+/, $line)[2] eq 'GPC1') { + $isGPC1 = 1; + } + } + } + if ($isGPC1 != 1) { + $burntoolState = 0; # If it's not GPC1, you shouldn't have run burntool. + } + elsif (($isGPC1 == 1) && ($burntoolState == 1)) { + my $ppConfigDump_cmd = "$ppConfigDump -camera GPC1 -dump-camera -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + IPC::Cmd::run(command => $ppConfigDump_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform ppConfigDump"); + exit($error_code); + } + + # This is ugly, but doing a full parse for one entry is a bit wasteful. + foreach my $line (split /\n/, (join "", @$stdout_buf)) { + if ($line =~ /BURNTOOL.STATE.GOOD/) { + $line =~ s/^\s+//; + $burntoolStateGood = (split /\s+/, $line)[2]; + last; + } + } + $burntoolState = $burntoolStateGood; # Positive because this has the header table. + } + $cmdflags .= " -burntool_state $burntoolState "; } @@ -123,4 +167,6 @@ $command .= " -tmp_class_id $tmp_class_id"; # the original class_id supplied by the user, replace by ppStats CLASS.ID $command .= " -uri $uri "; +$command .= " -bytes $bytes" if $bytes; +$command .= " -md5sum $md5sum" if $md5sum and ($md5sum ne 'NULL'); $command .= " -hostname $host" if defined $host; $command .= " -dbname $dbname" if defined $dbname; Index: branches/eam_branches/20090715/ippScripts/scripts/stack_skycell.pl =================================================================== --- branches/eam_branches/20090715/ippScripts/scripts/stack_skycell.pl (revision 25623) +++ branches/eam_branches/20090715/ippScripts/scripts/stack_skycell.pl (revision 25624) @@ -141,4 +141,6 @@ $ipprc->define_camera($camera); +print "I've configured everything: $stack_id\n"; + # Recipes to use based on reduction class $reduction = 'DEFAULT' unless defined $reduction; @@ -150,5 +152,18 @@ } -print "I've configured everything: $stack_id\n"; +my $recipe; +{ + my $command = "$ppConfigDump -camera $camera -recipe PPSTACK $recipe_ppStack -dump-recipe PPSTACK -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $stack_id, $error_code); + } + $recipe = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $stack_id, $PS_EXIT_PROG_ERROR); +} +my $convolve = metadataLookupBool($recipe, 'CONVOLVE'); # Convolve inputs? + # Generate MDC file with the inputs @@ -166,5 +181,5 @@ my $mask = $ipprc->filename( "PSWARP.OUTPUT.MASK", $file->{path_base} ); # Mask name my $weight = $ipprc->filename( "PSWARP.OUTPUT.VARIANCE", $file->{path_base} ); # Weight name - my $psf = $ipprc->filename( "PSPHOT.PSF.SKY.SAVE", $file->{path_base} ); # PSF name + my $psf = $convolve ? $ipprc->filename( "PSPHOT.PSF.SKY.SAVE", $file->{path_base} ) : undef; # PSF name my $sources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $file->{path_base}); # Sources name @@ -172,5 +187,5 @@ &my_die("Mask $mask does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $mask ); &my_die("Weight $weight does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $weight ); - &my_die("PSF $psf does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $psf ); + &my_die("PSF $psf does not exist", $stack_id, $PS_EXIT_SYS_ERROR) if ($convolve and not $ipprc->file_exists( $psf )); &my_die("Sources $sources does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $sources ); @@ -178,9 +193,6 @@ print $listFile "\tMASK\tSTR\t" . $mask . "\n"; print $listFile "\tVARIANCE\tSTR\t" . $weight . "\n"; - print $listFile "\tPSF\tSTR\t" . $psf . "\n"; + print $listFile "\tPSF\tSTR\t" . $psf . "\n" if $convolve; print $listFile "\tSOURCES\tSTR\t" . $sources . "\n"; - - ### XXX NEED TO UPDATE THESE appropriately - print $listFile "\tWEIGHTING\tF32\t" . 1.0 . "\n"; print $listFile "END\n\n"; @@ -274,5 +286,5 @@ # &my_die("Couldn't find expected output file: $bin2Name", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin2Name); } - + print "I've stacked $listName: $stack_id\n"; } @@ -352,20 +364,11 @@ print "Ensuring temporary images are deleted.\n"; - my $command = "$ppConfigDump -camera $camera -recipe PPSTACK $recipe_ppStack -dump-recipe PPSTACK -"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppConfigDump: $error_code", $stack_id, $error_code); - } - my $md = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $stack_id, $PS_EXIT_PROG_ERROR); - - my $temp_delete = metadataLookupBool($md, 'TEMP.DELETE'); # Delete temporary files? + + my $temp_delete = defined($recipe) and metadataLookupBool($recipe, 'TEMP.DELETE'); # Delete temp files? if ($temp_delete) { - my $temp_dir = metadataLookupStr($md, 'TEMP.DIR'); # Directory with temporary files - my $temp_image = metadataLookupStr($md, 'TEMP.IMAGE'); # Suffix for image - my $temp_mask = metadataLookupStr($md, 'TEMP.MASK'); # Suffix for mask - my $temp_weight = metadataLookupStr($md, 'TEMP.VARIANCE'); # Suffix for weight + my $temp_dir = metadataLookupStr($recipe, 'TEMP.DIR'); # Directory with temporary files + my $temp_image = metadataLookupStr($recipe, 'TEMP.IMAGE'); # Suffix for image + my $temp_mask = metadataLookupStr($recipe, 'TEMP.MASK'); # Suffix for mask + my $temp_weight = metadataLookupStr($recipe, 'TEMP.VARIANCE'); # Suffix for weight my $temp_root = basename($outroot); # Root name for temporary files Index: branches/eam_branches/20090715/ippScripts/scripts/summit_copy.pl =================================================================== --- branches/eam_branches/20090715/ippScripts/scripts/summit_copy.pl (revision 25623) +++ branches/eam_branches/20090715/ippScripts/scripts/summit_copy.pl (revision 25624) @@ -105,5 +105,5 @@ # if so pass the results to pztool to the results in pzDownloadImfile. # uncomment this to turn on -# my ($new_bytes, $new_md5) = check_instances($filename, $nebulous, $compress); +my ($new_bytes, $new_md5) = check_instances($filename, $nebulous, $compress); # command to update database @@ -121,5 +121,5 @@ # XXX: TODO: see above. Don't do this until pztool and the DB have # been updated -# $command .= " -md5sum $new_md5 -bytes $new_bytes"; +$command .= " -md5sum $new_md5 -bytes $new_bytes"; # update the database @@ -198,5 +198,5 @@ if ($b ne $new_bytes) { $error = "size of $instances[$i] does not match $instances[0]"; - } elsif ($m != $new_md5) { + } elsif ($m ne $new_md5) { $error = "md5sum of $instances[$i] does not match $instances[0]"; } Index: branches/eam_branches/20090715/ippScripts/scripts/warp_skycell.pl =================================================================== --- branches/eam_branches/20090715/ippScripts/scripts/warp_skycell.pl (revision 25623) +++ branches/eam_branches/20090715/ippScripts/scripts/warp_skycell.pl (revision 25624) @@ -183,4 +183,18 @@ close $astromFile; +# We need the recipe to determine if we care whether the PSF is generated or not +my $recipe; +{ + my $command = "$ppConfigDump -camera $camera -dump-recipe PSWARP -recipe PSWARP $recipe_pswarp -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR); + } + $recipe = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR); +} + # Run pswarp @@ -201,5 +215,4 @@ $command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID"; $command .= " -recipe PSWARP $recipe_pswarp"; - $command .= " -psf"; # Turn on PSF determination $command .= " -tracedest $traceDest -log $logDest"; $command .= " -threads $threads" if defined $threads; @@ -253,5 +266,5 @@ &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); - &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputPSF); + &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) if metadataLookupBool($recipe, 'PSF') and not $ipprc->file_exists($outputPSF); } Index: branches/eam_branches/20090715/ippTools/configure.ac =================================================================== --- branches/eam_branches/20090715/ippTools/configure.ac (revision 25623) +++ branches/eam_branches/20090715/ippTools/configure.ac (revision 25624) @@ -18,5 +18,5 @@ PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0]) PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) -PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.54]) +PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.55]) PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}" Index: branches/eam_branches/20090715/ippTools/share/Makefile.am =================================================================== --- branches/eam_branches/20090715/ippTools/share/Makefile.am (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/Makefile.am (revision 25624) @@ -110,5 +110,13 @@ disttool_definebyquery_stack.sql \ disttool_definebyquery_warp.sql \ - disttool_pendingcomponent.sql \ + disttool_defineinterest.sql \ + disttool_listinterests.sql \ + disttool_pending_camera.sql \ + disttool_pending_chip.sql \ + disttool_pending_diff.sql \ + disttool_pending_fake.sql \ + disttool_pending_raw.sql \ + disttool_pending_stack.sql \ + disttool_pending_warp.sql \ disttool_pendingfileset.sql \ disttool_pendingdest.sql \ @@ -116,6 +124,6 @@ disttool_queuercrun.sql \ disttool_revertrcrun.sql \ - disttool_revertrun_update.sql \ - disttool_revertrun_delete.sql \ + disttool_revertrun.sql \ + disttool_revertcomponent.sql \ disttool_revertfileset.sql \ disttool_toadvance.sql \ @@ -142,9 +150,9 @@ flatcorr_dropcamera.sql \ magictool_addmask.sql \ - magictool_censor_raw.sql \ - magictool_censor_chip.sql \ - magictool_censor_camera.sql \ - magictool_censor_warp.sql \ - magictool_censor_diff.sql \ + magictool_restore_camera.sql \ + magictool_restore_chip.sql \ + magictool_restore_diff.sql \ + magictool_restore_raw.sql \ + magictool_restore_warp.sql \ magictool_create_tmp_warpcomplete.sql \ magictool_definebyquery_insert.sql \ @@ -165,4 +173,5 @@ magictool_revertnode.sql \ magicdstool_completed_runs.sql \ + magicdstool_completedrevert.sql \ magicdstool_definebyquery_raw.sql \ magicdstool_definebyquery_chip.sql \ @@ -172,7 +181,11 @@ magicdstool_getrunids.sql \ magicdstool_getskycells.sql \ - magicdstool_todestreak.sql \ + magicdstool_revertdestreakedfile.sql \ + magicdstool_todestreak_camera.sql \ + magicdstool_todestreak_chip.sql \ + magicdstool_todestreak_diff.sql \ + magicdstool_todestreak_raw.sql \ + magicdstool_todestreak_warp.sql \ magicdstool_toremove.sql \ - magicdstool_torestore.sql \ magicdstool_torevert_raw.sql \ magicdstool_torevert_chip.sql \ Index: branches/eam_branches/20090715/ippTools/share/difftool_completed_runs.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/difftool_completed_runs.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/difftool_completed_runs.sql (revision 25624) @@ -1,19 +1,16 @@ -SELECT DISTINCT - diff_id, - all_magicked as magicked -FROM ( - SELECT - COUNT(diffInputSkyfile.skycell_id), COUNT(diffSkyfile.skycell_id), - diffSkyfile.*, - SUM(!diffSkyfile.magicked) = 0 as all_magicked - FROM diffRun - JOIN diffInputSkyfile USING(diff_id) - LEFT JOIN diffSkyfile USING(diff_id, skycell_id) - WHERE - diffRun.state = 'new' - GROUP BY - diffInputSkyfile.diff_id - HAVING - COUNT(diffInputSkyfile.skycell_id) = COUNT(diffSkyfile.skycell_id) - AND SUM(diffSkyfile.fault) = 0 - ) as Foo +SELECT + COUNT(diffInputSkyfile.skycell_id), + COUNT(diffSkyfile.skycell_id), + diffSkyfile.*, + SUM(!diffSkyfile.magicked) = 0 as all_magicked +FROM diffRun +JOIN diffInputSkyfile USING(diff_id) +LEFT JOIN diffSkyfile USING(diff_id, skycell_id) +WHERE + diffRun.state = 'new' +-- WHERE hook %s +GROUP BY + diffInputSkyfile.diff_id +HAVING + COUNT(diffInputSkyfile.skycell_id) = COUNT(diffSkyfile.skycell_id) + AND SUM(diffSkyfile.fault) = 0 Index: branches/eam_branches/20090715/ippTools/share/difftool_definewarpstack_part1.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/difftool_definewarpstack_part1.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/difftool_definewarpstack_part1.sql (revision 25624) @@ -27,5 +27,6 @@ ) AS diffExp USING(exp_id, warp_id) WHERE - warpSkyfile.fault = 0 + warpRun.state = 'full' + AND warpSkyfile.fault = 0 AND warpSkyfile.quality = 0 -- warp where hook %s Index: branches/eam_branches/20090715/ippTools/share/difftool_definewarpstack_part2.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/difftool_definewarpstack_part2.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/difftool_definewarpstack_part2.sql (revision 25624) @@ -30,4 +30,5 @@ WHERE warpSkyfile.warp_id = %lld + AND warpRun.state = 'full' AND warpSkyfile.fault = 0 AND warpSkyfile.quality = 0 Index: branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_camera.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_camera.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_camera.sql (revision 25624) @@ -3,4 +3,5 @@ camRun.cam_id as stage_id, rawExp.exp_name as run_tag, + camRun.magicked, distTarget.label, distTarget.target_id, Index: branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_chip.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_chip.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_chip.sql (revision 25624) @@ -2,4 +2,5 @@ 'chip' as stage, chipRun.chip_id as stage_id, + chipRun.magicked, rawExp.exp_name as run_tag, chipRun.label, Index: branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_diff.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_diff.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_diff.sql (revision 25624) @@ -2,4 +2,5 @@ 'diff' as stage, diffRun.diff_id AS stage_id, + diffRun.magicked, rawExp.exp_name as run_tag, distTarget.label, Index: branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_fake.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_fake.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_fake.sql (revision 25624) @@ -2,4 +2,5 @@ 'fake' as stage, fakeRun.fake_id AS stage_id, + CAST(0 AS SIGNED) AS magicked, rawExp.exp_name as run_tag, distTarget.label, Index: branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_raw.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_raw.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_raw.sql (revision 25624) @@ -3,4 +3,5 @@ rawExp.exp_id AS stage_id, rawExp.exp_name AS run_tag, + rawExp.magicked, distTarget.label, distTarget.target_id, Index: branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_stack.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_stack.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_stack.sql (revision 25624) @@ -2,4 +2,5 @@ 'stack' as stage, stackRun.stack_id AS stage_id, + CAST(0 AS SIGNED) AS magicked, -- run tag in the form 'stack.$skycell_id.$stack_id' CONCAT_WS('.', 'stack', stackRun.skycell_id, convert(stackRun.stack_id, CHAR)) as run_tag, @@ -8,4 +9,5 @@ distTarget.clean FROM stackRun +JOIN stackSumSkyfile USING(stack_id) JOIN distTarget ON distTarget.stage = 'stack' AND stackRun.label = distTarget.label @@ -19,2 +21,4 @@ AND distRun.dist_id IS NULL AND ((stackRun.state = 'full') OR (distTarget.clean AND stackRun.state = 'cleaned')) + -- we shouldn't need to check fault. If faulted it shouldn't be full + AND (stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0) Index: branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_warp.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_warp.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/disttool_definebyquery_warp.sql (revision 25624) @@ -2,4 +2,5 @@ 'warp' as stage, warpRun.warp_id AS stage_id, + warpRun.magicked, rawExp.exp_name as run_tag, distTarget.label, Index: branches/eam_branches/20090715/ippTools/share/disttool_defineinterest.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_defineinterest.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/disttool_defineinterest.sql (revision 25624) @@ -0,0 +1,6 @@ +INSERT INTO rcInterest + SELECT 0, dest_id, target_id, '@STATE@' + FROM distTarget + JOIN rcDestination + LEFT JOIN rcInterest USING(dest_id, target_id) + WHERE (int_id IS NULL) Index: branches/eam_branches/20090715/ippTools/share/disttool_listinterests.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_listinterests.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/disttool_listinterests.sql (revision 25624) @@ -0,0 +1,13 @@ +SELECT + int_id, + rcInterest.state AS state, + dest_id, + name AS dest_name, + rcDestination.state as dest_state, + label, + filter, + stage, + distTarget.state as target_state +FROM rcInterest + JOIN distTarget USING(target_id) + JOIN rcDestination USING(dest_id) Index: branches/eam_branches/20090715/ippTools/share/disttool_pending_camera.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_pending_camera.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/disttool_pending_camera.sql (revision 25624) @@ -0,0 +1,31 @@ +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + 'exposure' AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + camProcessedExp.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + camRun.state, + camRun.state AS data_state, + camProcessedExp.quality, + distRun.no_magic, + chipRun.magicked +FROM distRun +JOIN camRun ON camRun.cam_id = distRun.stage_id +JOIN camProcessedExp USING(cam_id) +JOIN chipRun USING(chip_id) +-- JOIN chipProcessedImfile USING(exp_id, chip_id) +JOIN rawExp using(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id +-- AND chipProcessedImfile.class_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'camera' + AND distComponent.dist_id IS NULL + AND (distRun.clean OR (chipRun.magicked AND camRun.magicked) OR distRun.no_magic) + AND (camRun.state = 'full' OR (distRun.clean AND camRun.state = 'cleaned')) Index: branches/eam_branches/20090715/ippTools/share/disttool_pending_chip.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_pending_chip.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/disttool_pending_chip.sql (revision 25624) @@ -0,0 +1,29 @@ +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + chipProcessedImfile.class_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + chipProcessedImfile.path_base, + chipProcessedImfile.path_base as chip_path_base, + chipRun.state, + chipProcessedImfile.data_state, + chipProcessedImfile.quality, + distRun.no_magic, + chipProcessedImfile.magicked +FROM distRun +JOIN chipRun ON chipRun.chip_id = distRun.stage_id +JOIN rawExp using(exp_id) +JOIN chipProcessedImfile ON chipProcessedImfile.chip_id = stage_id +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND chipProcessedImfile.class_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'chip' + AND distComponent.dist_id IS NULL + AND (distRun.clean OR chipRun.magicked OR distRun.no_magic) + AND (chipRun.state = 'full' OR (distRun.clean AND chipRun.state = 'cleaned')) Index: branches/eam_branches/20090715/ippTools/share/disttool_pending_diff.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_pending_diff.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/disttool_pending_diff.sql (revision 25624) @@ -0,0 +1,37 @@ +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + diffSkyfile.skycell_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + diffSkyfile.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + diffRun.state, + -- data_state doesn't exist yet + -- diffSkyfile.data_state, + 'full' AS data_state, + diffSkyfile.quality, + distRun.no_magic, + diffSkyfile.magicked +FROM distRun +JOIN diffRun ON stage_id = diff_id +JOIN diffSkyfile using(diff_id) +JOIN diffInputSkyfile USING(diff_id, skycell_id) +JOIN warpRun + ON warpRun.warp_id = diffInputSkyfile.warp1 +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND diffSkyfile.skycell_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'diff' + AND distComponent.dist_id IS NULL + AND (distRun.clean OR diffRun.magicked OR distRun.no_magic) + AND (diffRun.state = 'full' OR (distRun.clean AND diffRun.state = 'cleaned')) Index: branches/eam_branches/20090715/ippTools/share/disttool_pending_fake.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_pending_fake.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/disttool_pending_fake.sql (revision 25624) @@ -0,0 +1,29 @@ +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + fakeProcessedImfile.class_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + fakeProcessedImfile.path_base, + CAST(NULL AS CHAR(255)) AS chip_path_base, + fakeRun.state, + fakeRun.state AS data_state, + 0 as quality, + distRun.no_magic, + 0 +FROM distRun +JOIN fakeRun ON fakeRun.fake_id = distRun.stage_id +JOIN fakeProcessedImfile USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id, exp_id) +JOIN rawExp using(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND fakeProcessedImfile.class_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'fake' + AND distComponent.dist_id IS NULL Index: branches/eam_branches/20090715/ippTools/share/disttool_pending_raw.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_pending_raw.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/disttool_pending_raw.sql (revision 25624) @@ -0,0 +1,58 @@ +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + rawImfile.class_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + rawImfile.uri as path_base, -- change this once rawImfile has path_base + chipProcessedImfile.path_base as chip_path_base, + CAST(NULL AS CHAR(255)) AS state, + CAST(NULL AS CHAR(255)) AS data_state, + 0 as quality, + distRun.no_magic, + rawImfile.magicked +FROM distRun +JOIN rawExp ON rawExp.exp_id = stage_id +JOIN rawImfile USING(exp_id) +JOIN chipRun ON chipRun.exp_id = rawExp.exp_id and chipRun.magicked +JOIN chipProcessedImfile + USING(chip_id, class_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND rawImfile.class_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.clean = 0 + AND distRun.stage = 'raw' + AND distComponent.dist_id IS NULL + AND (rawExp.magicked OR distRun.no_magic) +UNION + -- raw stage clean (dbinfo only) +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + 'exposure' AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + CAST(NULL AS CHAR(255)) AS path_base, + CAST(NULL AS CHAR(255)) AS chip_path_base, + CAST(NULL AS CHAR(255)) AS state, + CAST(NULL AS CHAR(255)) AS data_state, + 0 as quality, + distRun.no_magic, + rawExp.magicked +FROM distRun +JOIN rawExp ON exp_id = stage_id +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id +WHERE + distRun.state = 'new' + AND distRun.stage = 'raw' + AND distRun.clean + AND distComponent.dist_id IS NULL Index: branches/eam_branches/20090715/ippTools/share/disttool_pending_stack.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_pending_stack.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/disttool_pending_stack.sql (revision 25624) @@ -0,0 +1,48 @@ +SELECT DISTINCT + distRun.dist_id, + distRun.label, + stage, + stage_id, + stackRun.skycell_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + stackSumSkyfile.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + stackRun.state, + -- data_state doesn't exist yet + -- stackSumSkyfile.data_state, + 'full' AS data_state, + stackSumSkyfile.quality, + 1 AS no_magic, + 0 AS magicked +FROM distRun +JOIN stackRun + ON stage_id = stack_id +JOIN stackSumSkyfile + USING(stack_id) +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpSkyfile + ON stackInputSkyfile.warp_id = warpSkyfile.warp_id + AND stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.tess_id = warpSkyfile.tess_id +JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN rawExp + USING (exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND stackRun.skycell_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'stack' + AND distComponent.dist_id IS NULL + AND (stackRun.state = 'full' OR (distRun.clean AND stackRun.state = 'cleaned')) + AND (stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0) Index: branches/eam_branches/20090715/ippTools/share/disttool_pending_warp.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_pending_warp.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/disttool_pending_warp.sql (revision 25624) @@ -0,0 +1,32 @@ +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + warpSkyfile.skycell_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + warpSkyfile.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + warpRun.state, + warpSkyfile.data_state, + warpSkyfile.quality, + distRun.no_magic, + warpSkyfile.magicked +FROM distRun +JOIN warpRun ON stage_id = warp_id +JOIN warpSkyfile using(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun ON camRun.chip_id = chipRun.chip_id +JOIN rawExp using(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND warpSkyfile.skycell_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'warp' + AND distComponent.dist_id IS NULL + AND (distRun.clean OR warpRun.magicked OR distRun.no_magic) + AND (warpRun.state = 'full' OR (distRun.clean AND warpRun.state = 'cleaned')) Index: branches/eam_branches/20090715/ippTools/share/disttool_pendingcomponent.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_pendingcomponent.sql (revision 25623) +++ (revision ) @@ -1,283 +1,0 @@ -SELECT * FROM ( - -- raw stage -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - rawImfile.class_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - rawImfile.uri as path_base, -- change this once rawImfile has path_base - chipProcessedImfile.path_base as chip_path_base, - NULL as state, - NULL as data_state, - 0 as quality, - distRun.no_magic, - rawImfile.magicked -FROM distRun -JOIN rawExp ON rawExp.exp_id = stage_id -JOIN rawImfile USING(exp_id) -JOIN chipRun ON chipRun.exp_id = rawExp.exp_id and chipRun.magicked -JOIN chipProcessedImfile - USING(chip_id, class_id) -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id - AND rawImfile.class_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.clean = 0 - AND distRun.stage = 'raw' - AND distComponent.dist_id IS NULL - AND (rawExp.magicked OR distRun.no_magic) - -- where hook 1 %s -UNION - -- raw stage clean (dbinfo only) -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - 'exposure' AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - NULL, - NULL, - NULL as state, - NULL as data_state, - 0 as quality, - distRun.no_magic, - rawExp.magicked -FROM distRun -JOIN rawExp ON exp_id = stage_id -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id -WHERE - distRun.state = 'new' - AND distRun.stage = 'raw' - AND distRun.clean - AND distComponent.dist_id IS NULL - -- where hook 2 %s - --- chip stage -UNION -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - chipProcessedImfile.class_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - chipProcessedImfile.path_base, - chipProcessedImfile.path_base as chip_path_base, - chipRun.state, - chipProcessedImfile.data_state, - chipProcessedImfile.quality, - distRun.no_magic, - chipProcessedImfile.magicked -FROM distRun -JOIN chipRun ON chipRun.chip_id = distRun.stage_id -JOIN rawExp using(exp_id) -JOIN chipProcessedImfile ON chipProcessedImfile.chip_id = stage_id -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id - AND chipProcessedImfile.class_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.stage = 'chip' - AND distComponent.dist_id IS NULL - AND (distRun.clean OR chipRun.magicked OR distRun.no_magic) - AND (chipRun.state = 'full' OR (distRun.clean AND chipRun.state = 'cleaned')) - -- where hook 3 %s -UNION -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - 'exposure' AS component, --- chipProcessedImfile.class_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - camProcessedExp.path_base, - NULL as chip_path_base, - camRun.state, - NULL, - camProcessedExp.quality, - distRun.no_magic, - chipRun.magicked -FROM distRun -JOIN camRun ON camRun.cam_id = distRun.stage_id -JOIN camProcessedExp USING(cam_id) -JOIN chipRun USING(chip_id) --- JOIN chipProcessedImfile USING(exp_id, chip_id) -JOIN rawExp using(exp_id) -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id --- AND chipProcessedImfile.class_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.stage = 'camera' - AND distComponent.dist_id IS NULL - AND (distRun.clean OR (chipRun.magicked AND camRun.magicked) OR distRun.no_magic) - AND (camRun.state = 'full' OR (distRun.clean AND camRun.state = 'cleaned')) - -- where hook 4 %s -UNION -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - fakeProcessedImfile.class_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - fakeProcessedImfile.path_base, - NULL, - fakeRun.state, - NULL, - 0 as quality, - distRun.no_magic, - 0 -FROM distRun -JOIN fakeRun ON fakeRun.fake_id = distRun.stage_id -JOIN fakeProcessedImfile USING(fake_id) -JOIN camRun USING(cam_id) -JOIN chipRun USING(chip_id, exp_id) -JOIN rawExp using(exp_id) -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id - AND fakeProcessedImfile.class_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.stage = 'fake' - AND distComponent.dist_id IS NULL - -- where hook 5 %s -UNION -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - warpSkyfile.skycell_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - warpSkyfile.path_base, - NULL as chip_path_base, - warpRun.state, - warpSkyfile.data_state, - warpSkyfile.quality, - distRun.no_magic, - warpSkyfile.magicked -FROM distRun -JOIN warpRun ON stage_id = warp_id -JOIN warpSkyfile using(warp_id) -JOIN fakeRun USING(fake_id) -JOIN camRun USING(cam_id) -JOIN chipRun ON camRun.chip_id = chipRun.chip_id -JOIN rawExp using(exp_id) -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id - AND warpSkyfile.skycell_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.stage = 'warp' - AND distComponent.dist_id IS NULL - AND (distRun.clean OR warpRun.magicked OR distRun.no_magic) - AND (warpRun.state = 'full' OR (distRun.clean AND warpRun.state = 'cleaned')) - -- where hook 6 %s -UNION -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - diffSkyfile.skycell_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - diffSkyfile.path_base, - NULL as chip_path_base, - diffRun.state, - -- data_state doesn't exist yet - -- diffSkyfile.data_state, - 'full' AS data_state, - diffSkyfile.quality, - distRun.no_magic, - diffSkyfile.magicked -FROM distRun -JOIN diffRun ON stage_id = diff_id -JOIN diffSkyfile using(diff_id) -JOIN diffInputSkyfile USING(diff_id, skycell_id) -JOIN warpRun - ON warpRun.warp_id = diffInputSkyfile.warp1 -JOIN fakeRun USING(fake_id) -JOIN camRun USING(cam_id) -JOIN chipRun USING(chip_id) -JOIN rawExp USING(exp_id) -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id - AND diffSkyfile.skycell_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.stage = 'diff' - AND distComponent.dist_id IS NULL - AND (distRun.clean OR diffRun.magicked OR distRun.no_magic) - AND (diffRun.state = 'full' OR (distRun.clean AND diffRun.state = 'cleaned')) - -- where hook 7 %s -UNION -SELECT DISTINCT - distRun.dist_id, - distRun.label, - stage, - stage_id, - stackRun.skycell_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - stackSumSkyfile.path_base, - NULL as chip_path_base, - stackRun.state, - -- data_state doesn't exist yet - -- stackSumSkyfile.data_state, - 'full' AS data_state, - stackSumSkyfile.quality, - 1 AS no_magic, - 0 AS magicked -FROM distRun -JOIN stackRun - ON stage_id = stack_id -JOIN stackSumSkyfile - USING(stack_id) -JOIN stackInputSkyfile - USING(stack_id) -JOIN warpSkyfile - ON stackInputSkyfile.warp_id = warpSkyfile.warp_id - AND stackRun.skycell_id = warpSkyfile.skycell_id - AND stackRun.tess_id = warpSkyfile.tess_id -JOIN warpRun - ON warpRun.warp_id = warpSkyfile.warp_id -JOIN fakeRun - USING(fake_id) -JOIN camRun - USING(cam_id) -JOIN chipRun - ON camRun.chip_id = chipRun.chip_id -JOIN rawExp - USING (exp_id) -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id - AND stackRun.skycell_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.stage = 'stack' - AND distComponent.dist_id IS NULL - AND (stackRun.state = 'full' OR (distRun.clean AND stackRun.state = 'cleaned')) - -- where hook 8 %s -) as Foo Index: branches/eam_branches/20090715/ippTools/share/disttool_pendingfileset.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_pendingfileset.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/disttool_pendingfileset.sql (revision 25624) @@ -5,14 +5,13 @@ stage_id, CONCAT_WS('.', distRun.outroot, CONVERT(distRun.dist_id, CHAR)) as dist_dir, - rcDSProduct.name AS product_name, - rcDSProduct.prod_id, - rcDSProduct.dbname AS ds_dbname, - rcDSProduct.dbhost AS ds_dbhost + rcDestination.name AS product_name, + rcDestination.dest_id, + rcDestination.dbname AS ds_dbname, + rcDestination.dbhost AS ds_dbhost FROM rcDestination JOIN rcInterest USING(dest_id) JOIN distTarget USING(target_id) -JOIN rcDSProduct USING(prod_id) JOIN distRun USING(target_id) -LEFT JOIN rcDSFileset USING(prod_id, dist_id) +LEFT JOIN rcDSFileset USING(dest_id, dist_id) WHERE distRun.state = 'full' AND rcDestination.state = 'enabled' Index: branches/eam_branches/20090715/ippTools/share/disttool_queuercrun.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_queuercrun.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/disttool_queuercrun.sql (revision 25624) @@ -13,7 +13,6 @@ JOIN rcInterest USING(dest_id) JOIN distTarget USING(target_id) -JOIN rcDSProduct USING(prod_id) JOIN distRun USING(target_id, stage) -JOIN rcDSFileset USING(prod_id, dist_id) +JOIN rcDSFileset USING(dest_id, dist_id) LEFT JOIN rcRun using(fs_id, dest_id) WHERE rcRun.rc_id IS NULL Index: branches/eam_branches/20090715/ippTools/share/disttool_revertcomponent.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_revertcomponent.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/disttool_revertcomponent.sql (revision 25624) @@ -0,0 +1,5 @@ +DELETE FROM distComponent +USING distComponent, distRun +WHERE distComponent.dist_id = distRun.dist_id + AND distRun.state = 'new' + AND distComponent.fault != 0 Index: branches/eam_branches/20090715/ippTools/share/disttool_revertrun.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_revertrun.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/disttool_revertrun.sql (revision 25624) @@ -0,0 +1,4 @@ +UPDATE distRun +SET distRun.fault = 0 +WHERE distRun.state = 'new' + AND distRun.fault != 0 Index: branches/eam_branches/20090715/ippTools/share/disttool_revertrun_delete.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_revertrun_delete.sql (revision 25623) +++ (revision ) @@ -1,5 +1,0 @@ -DELETE FROM distComponent -USING distComponent, distRun -WHERE distComponent.dist_id = distRun.dist_id - AND distRun.state = 'new' - AND distComponent.fault != 0 Index: branches/eam_branches/20090715/ippTools/share/disttool_revertrun_update.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_revertrun_update.sql (revision 25623) +++ (revision ) @@ -1,4 +1,0 @@ -UPDATE distRun -SET distRun.fault = 0 -WHERE distRun.state = 'new' - AND distRun.fault != 0 Index: branches/eam_branches/20090715/ippTools/share/disttool_updatercrun.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/disttool_updatercrun.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/disttool_updatercrun.sql (revision 25624) @@ -1,6 +1,5 @@ UPDATE rcRun JOIN rcDestination USING(dest_id) -JOIN rcDSProduct USING(prod_id) -JOIN rcDSFileset using(prod_id, fs_id) +JOIN rcDSFileset using(dest_id, fs_id) SET -- set hook %s Index: branches/eam_branches/20090715/ippTools/share/magicdstool_completed_runs.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_completed_runs.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_completed_runs.sql (revision 25624) @@ -1,9 +1,11 @@ SELECT DISTINCT - magic_ds_id + magic_ds_id, + re_place, + label FROM ( -- raw stage SELECT - magicDSRun.magic_ds_id + magicDSRun.* FROM magicDSRun JOIN rawImfile ON stage_id = rawImfile.exp_id @@ -23,5 +25,5 @@ -- chip stage SELECT - magicDSRun.magic_ds_id + magicDSRun.* FROM magicDSRun JOIN chipProcessedImfile ON stage_id = chip_id @@ -42,5 +44,5 @@ -- camera stage SELECT - magicDSRun.magic_ds_id + magicDSRun.* FROM magicDSRun JOIN camProcessedExp ON stage_id = camProcessedExp.cam_id @@ -59,5 +61,5 @@ -- warp stage SELECT - magicDSRun.magic_ds_id + magicDSRun.* FROM magicDSRun JOIN warpSkyfile on stage_id = warp_id @@ -79,5 +81,5 @@ -- diff stage SELECT DISTINCT - magicDSRun.magic_ds_id + magicDSRun.* FROM magicDSRun JOIN magicRun USING (magic_id) @@ -101,3 +103,3 @@ AND SUM(magicDSFile.fault) = 0 - ) as Foo + ) as magicDSRun Index: branches/eam_branches/20090715/ippTools/share/magicdstool_completedrevert.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_completedrevert.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_completedrevert.sql (revision 25624) @@ -0,0 +1,10 @@ +SELECT + magic_ds_id, + state +FROM magicDSRun +LEFT JOIN magicDSFile USING(magic_ds_id) +WHERE + magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored' +GROUP BY magic_ds_id +HAVING COUNT(component) = 0 + Index: branches/eam_branches/20090715/ippTools/share/magicdstool_revertdestreakedfile.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_revertdestreakedfile.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_revertdestreakedfile.sql (revision 25624) @@ -0,0 +1,7 @@ +DELETE FROM magicDSFile USING magicDSFile, magicDSRun +WHERE (magicDSRun.magic_ds_id = magicDSFile.magic_ds_id) + AND ((magicDSRun.state = 'new' AND magicDSFile.fault != 0) + OR (magicDSRun.state = 'goto_restored' + OR magicDSRun.state = 'goto_censored' + ) + ) Index: branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak.sql (revision 25623) +++ (revision ) @@ -1,228 +1,0 @@ -SELECT * -FROM ( -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicRun.magic_id, - magicRun.exp_id, - magicDSRun.label, - camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - stage_id, - class_id as component, - rawImfile.uri as uri, - NULL as path_base, - magicRun.inverse, - camProcessedExp.path_base as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM magicDSRun -JOIN magicMask USING (magic_id) -JOIN magicRun USING (magic_id) -JOIN camProcessedExp USING(cam_id) -JOIN rawImfile ON magicRun.exp_id = rawImfile.exp_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = rawImfile.class_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'raw' - AND magicDSFile.component IS NULL -UNION - -- chipProcesedImfiles -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicDSRun.magic_id, - chipRun.exp_id, - magicDSRun.label, - camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - stage_id, - class_id as component, - chipProcessedImfile.uri, - chipProcessedImfile.path_base, - magicRun.inverse, - camProcessedExp.path_base as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM magicDSRun -JOIN magicMask USING (magic_id) -JOIN magicRun USING(magic_id) -JOIN camProcessedExp USING(cam_id) -JOIN chipRun ON chip_id = stage_id -JOIN chipProcessedImfile USING(chip_id) -JOIN rawExp ON chipRun.exp_id = rawExp.exp_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = chipProcessedImfile.class_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'chip' - AND chipRun.state = 'full' - AND chipProcessedImfile.fault = 0 - AND chipProcessedImfile.quality = 0 - AND magicDSFile.component IS NULL -UNION - -- camProcessedExp -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicDSRun.magic_id, - chipRun.exp_id, - magicDSRun.label, - camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - stage_id, - 'exposure' as component, - NULL AS uri, - camProcessedExp.path_base, - magicRun.inverse, - camProcessedExp.path_base as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM magicDSRun -JOIN magicMask USING (magic_id) -JOIN magicRun USING(magic_id) -JOIN camRun ON magicDSRun.stage_id = camRun.cam_id -JOIN camProcessedExp ON camRun.cam_id = camProcessedExp.cam_id -JOIN chipRun USING(chip_id) -JOIN rawExp ON chipRun.exp_id = rawExp.exp_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'camera' - AND camRun.state = 'full' - AND chipRun.state = 'full' - AND chipRun.magicked - AND camProcessedExp.fault = 0 - AND camProcessedExp.quality = 0 - AND magicDSFile.component IS NULL -UNION --- warpSkyfiles -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicRun.magic_id, - magicRun.exp_id, - magicDSRun.label, - camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - stage_id, - warpSkyfile.skycell_id as component, - warpSkyfile.uri, - warpSkyfile.path_base, - magicRun.inverse, - NULL as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM magicDSRun -JOIN magicMask USING (magic_id) -JOIN magicRun USING (magic_id) -JOIN warpRun ON warp_id = stage_id -JOIN warpSkyfile USING(warp_id) -JOIN rawExp ON magicRun.exp_id = rawExp.exp_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = warpSkyfile.skycell_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'warp' - AND warpRun.state = 'full' - AND warpSkyfile.fault = 0 - AND warpSkyfile.quality = 0 - AND magicDSFile.component IS NULL -UNION --- regular diffSkyfiles -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicRun.magic_id, - magicRun.exp_id, - magicDSRun.label, - rawExp.camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - magicRun.diff_id as stage_id, - diffSkyfile.skycell_id as component, - NULL AS uri, - diffSkyfile.path_base, - magicRun.inverse, - NULL as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM rawExp -JOIN magicRun USING (exp_id) -JOIN magicMask USING (magic_id) -JOIN magicDSRun USING(magic_id) -JOIN magicInputSkyfile USING(magic_id) -JOIN diffRun USING(diff_id) -JOIN diffSkyfile - ON magicRun.diff_id = diffSkyfile.diff_id - AND magicInputSkyfile.node = diffSkyfile.skycell_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = diffSkyfile.skycell_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'diff' - AND diffRun.bothways = 0 - AND diffSkyfile.fault = 0 - AND diffSkyfile.quality = 0 - AND magicDSFile.component IS NULL --- bothways diffSkyfiles -UNION -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicRun.magic_id, - magicRun.exp_id, - magicDSRun.label, - rawExp.camera, - magicMask.uri as streaks_uri, - (SELECT uri from magicMask where magic_id = inv_magic_id) AS inv_streaks_uri, - stage, - magicRun.diff_id as stage_id, - diffSkyfile.skycell_id as component, - NULL AS uri, - diffSkyfile.path_base, - magicRun.inverse, - NULL as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM rawExp -JOIN magicRun USING (exp_id) -JOIN magicMask USING (magic_id) -JOIN magicDSRun USING(magic_id) -JOIN magicInputSkyfile USING(magic_id) -JOIN diffRun USING(diff_id) -JOIN diffSkyfile - ON magicRun.diff_id = diffSkyfile.diff_id - AND magicInputSkyfile.node = diffSkyfile.skycell_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = diffSkyfile.skycell_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'diff' - AND diffRun.bothways - AND diffSkyfile.fault = 0 - AND diffSkyfile.quality = 0 - AND magicDSFile.component IS NULL -) as Foo Index: branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak_camera.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak_camera.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak_camera.sql (revision 25624) @@ -0,0 +1,37 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicDSRun.magic_id, + chipRun.exp_id, + magicDSRun.label, + camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + stage_id, + 'exposure' as component, + CAST(NULL AS CHAR(255)) AS uri, + camProcessedExp.path_base, + magicRun.inverse, + camProcessedExp.path_base as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING(magic_id) +JOIN camRun ON magicDSRun.stage_id = camRun.cam_id +JOIN camProcessedExp ON camRun.cam_id = camProcessedExp.cam_id +JOIN chipRun USING(chip_id) +JOIN rawExp ON chipRun.exp_id = rawExp.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'camera' + AND camRun.state = 'full' + AND chipRun.state = 'full' + AND chipRun.magicked + AND camProcessedExp.fault = 0 + AND camProcessedExp.quality = 0 + AND magicDSFile.component IS NULL Index: branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak_chip.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak_chip.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak_chip.sql (revision 25624) @@ -0,0 +1,36 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicDSRun.magic_id, + chipRun.exp_id, + magicDSRun.label, + camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + stage_id, + class_id as component, + chipProcessedImfile.uri, + chipProcessedImfile.path_base, + magicRun.inverse, + camProcessedExp.path_base as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING(magic_id) +JOIN camProcessedExp USING(cam_id) +JOIN chipRun ON chip_id = stage_id +JOIN chipProcessedImfile USING(chip_id) +JOIN rawExp ON chipRun.exp_id = rawExp.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = chipProcessedImfile.class_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'chip' + AND chipRun.state = 'full' + AND chipProcessedImfile.fault = 0 + AND chipProcessedImfile.quality = 0 + AND magicDSFile.component IS NULL Index: branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak_diff.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak_diff.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak_diff.sql (revision 25624) @@ -0,0 +1,82 @@ +SELECT * FROM ( +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicRun.exp_id, + magicDSRun.label, + rawExp.camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + magicRun.diff_id as stage_id, + diffSkyfile.skycell_id as component, + CAST(NULL AS CHAR(255)) AS uri, + diffSkyfile.path_base, + magicRun.inverse, + CAST(NULL AS CHAR(255)) as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM rawExp +JOIN magicRun USING (exp_id) +JOIN magicMask USING (magic_id) +JOIN magicDSRun USING(magic_id) +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffSkyfile + ON magicRun.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = diffSkyfile.skycell_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'diff' + AND diffRun.bothways = 0 + AND diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 + AND magicDSFile.component IS NULL +-- bothways diffSkyfiles +UNION +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicRun.exp_id, + magicDSRun.label, + rawExp.camera, + magicMask.uri as streaks_uri, + (SELECT uri from magicMask where magic_id = inv_magic_id) AS inv_streaks_uri, + stage, + magicRun.diff_id as stage_id, + diffSkyfile.skycell_id as component, + CAST(NULL AS CHAR(255)) AS uri, + diffSkyfile.path_base, + magicRun.inverse, + CAST(NULL AS CHAR(255)) as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM rawExp +JOIN magicRun USING (exp_id) +JOIN magicMask USING (magic_id) +JOIN magicDSRun USING(magic_id) +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffSkyfile + ON magicRun.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = diffSkyfile.skycell_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'diff' + AND diffRun.bothways + AND diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 + AND magicDSFile.component IS NULL +) as magicDSRun +-- we need the following so this query is compatible with the other stages +WHERE magic_ds_id IS NOT NULL Index: branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak_raw.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak_raw.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak_raw.sql (revision 25624) @@ -0,0 +1,31 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicRun.exp_id, + magicDSRun.label, + camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + stage_id, + class_id as component, + rawImfile.uri as uri, + CAST(NULL AS CHAR(255)) as path_base, + magicRun.inverse, + camProcessedExp.path_base as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING (magic_id) +JOIN camProcessedExp USING(cam_id) +JOIN rawImfile ON magicRun.exp_id = rawImfile.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = rawImfile.class_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'raw' + AND magicDSFile.component IS NULL Index: branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak_warp.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak_warp.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak_warp.sql (revision 25624) @@ -0,0 +1,35 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicRun.exp_id, + magicDSRun.label, + camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + stage_id, + warpSkyfile.skycell_id as component, + warpSkyfile.uri, + warpSkyfile.path_base, + magicRun.inverse, + CAST(NULL AS CHAR(255)) as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING (magic_id) +JOIN warpRun ON warp_id = stage_id +JOIN warpSkyfile USING(warp_id) +JOIN rawExp ON magicRun.exp_id = rawExp.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = warpSkyfile.skycell_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'warp' + AND warpRun.state = 'full' + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 + AND magicDSFile.component IS NULL Index: branches/eam_branches/20090715/ippTools/share/magicdstool_torestore.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_torestore.sql (revision 25623) +++ (revision ) @@ -1,12 +1,0 @@ -SELECT - magic_ds_id, - stage, - stage_id - component, - backup_path_base, - recovery_path_base -FROM magicDSFile -JOIN magicDSRun USING(magic_ds_id) -WHERE magicDSRun.state = 'goto_restored' - AND backup_path_base IS NOT NULL - AND recovery_path_base IS NOT NULL Index: branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_camera.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_camera.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_camera.sql (revision 25624) @@ -1,4 +1,5 @@ SELECT magic_ds_id, + magicDSRun.state, exp_id, re_place, @@ -19,4 +20,7 @@ JOIN rawExp using(exp_id) WHERE magicDSRun.stage = 'camera' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR ((magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored') + AND (backup_path_base IS NOT NULL) + ) + ) Index: branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_chip.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_chip.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_chip.sql (revision 25624) @@ -1,4 +1,5 @@ SELECT magic_ds_id, + magicDSRun.state, exp_id, re_place, @@ -18,4 +19,7 @@ JOIN rawExp using(exp_id) WHERE magicDSRun.stage = 'chip' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR ((magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored') + AND ((backup_path_base IS NOT NULL) OR (recovery_path_base IS NOT NULL)) + ) + ) Index: branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_diff.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_diff.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_diff.sql (revision 25624) @@ -1,4 +1,5 @@ SELECT magic_ds_id, + magicDSRun.state, exp_id, re_place, @@ -18,4 +19,7 @@ JOIN rawExp USING(exp_id) WHERE magicDSRun.stage = 'diff' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR ((magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored') + AND ((backup_path_base IS NOT NULL) OR (recovery_path_base IS NOT NULL)) + ) + ) Index: branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_raw.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_raw.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_raw.sql (revision 25624) @@ -1,4 +1,5 @@ SELECT magic_ds_id, + magicDSRun.state, exp_id, re_place, @@ -15,7 +16,8 @@ JOIN magicDSFile using(magic_ds_id) JOIN rawImfile ON (stage_id = exp_id AND component = class_id) - JOIN summitImfile ON (rawImfile.exp_name = summitImfile.exp_name - AND rawImfile.tmp_class_id = summitImfile.class_id) WHERE magicDSRun.stage = 'raw' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR ((magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored') + AND ((backup_path_base IS NOT NULL) OR (recovery_path_base IS NOT NULL)) + ) + ) Index: branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_warp.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_warp.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_warp.sql (revision 25624) @@ -1,4 +1,5 @@ SELECT magic_ds_id, + magicDSRun.state, exp_id, re_place, @@ -21,4 +22,7 @@ JOIN rawExp USING(exp_id) WHERE magicDSRun.stage = 'warp' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR ((magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored') + AND ((backup_path_base IS NOT NULL) OR (recovery_path_base IS NOT NULL)) + ) + ) Index: branches/eam_branches/20090715/ippTools/share/magictool_censor_camera.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magictool_censor_camera.sql (revision 25623) +++ (revision ) @@ -1,8 +1,0 @@ -UPDATE magicRun - JOIN magicDSRun USING(magic_id) - JOIN magicDSFile USING(magic_ds_id) - JOIN camRun ON magic_ds_id = magicked -SET camRun.magicked = 0, - magicDSRun.state = 'censored', - magicDSFile.fault = 42 - Index: branches/eam_branches/20090715/ippTools/share/magictool_censor_chip.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magictool_censor_chip.sql (revision 25623) +++ (revision ) @@ -1,9 +1,0 @@ -UPDATE chipRun - JOIN chipProcessedImfile USING(chip_id, exp_id) - JOIN magicDSRun ON chipRun.magicked = magicDSRun.magic_ds_id - JOIN magicDSFile USING(magic_ds_id) - JOIN magicRun USING(magic_id, exp_id) -SET chipProcessedImfile.magicked = 0, - chipRun.magicked = 0, - magicDSRun.state = 'censored', - magicDSFile.fault = 42 Index: branches/eam_branches/20090715/ippTools/share/magictool_censor_diff.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magictool_censor_diff.sql (revision 25623) +++ (revision ) @@ -1,9 +1,0 @@ -UPDATE magicRun - JOIN magicDSRun USING(magic_id) - JOIN magicDSFile USING(magic_ds_id) - JOIN diffRun ON magic_ds_id = magicked - JOIN diffSkyfile ON diffRun.diff_id = diffSkyfile.diff_id -SET diffRun.magicked = 0, - diffSkyfile.magicked = 0, - magicDSRun.state = 'censored', - magicDSFile.fault = 42 Index: branches/eam_branches/20090715/ippTools/share/magictool_censor_raw.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magictool_censor_raw.sql (revision 25623) +++ (revision ) @@ -1,10 +1,0 @@ -UPDATE magicRun - JOIN magicDSRun USING(magic_id) - JOIN magicDSFile USING(magic_ds_id) - JOIN rawExp ON rawExp.magicked = magic_ds_id - JOIN rawImfile ON rawExp.exp_id = rawImfile.exp_id -SET rawImfile.magicked = 0, - rawExp.magicked = 0, - magicDSRun.state = 'censored', - magicDSFile.fault = 42 - Index: branches/eam_branches/20090715/ippTools/share/magictool_censor_warp.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magictool_censor_warp.sql (revision 25623) +++ (revision ) @@ -1,9 +1,0 @@ -UPDATE magicRun - JOIN magicDSRun USING(magic_id) - JOIN magicDSFile USING(magic_ds_id) - JOIN warpRun ON magic_ds_id = magicked - JOIN warpSkyfile using(warp_id) -SET warpRun.magicked = 0, - warpSkyfile.magicked = 0, - magicDSRun.state = 'censored', - magicDSFile.fault = 42 Index: branches/eam_branches/20090715/ippTools/share/magictool_restore_camera.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magictool_restore_camera.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/magictool_restore_camera.sql (revision 25624) @@ -0,0 +1,6 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN camRun ON stage = 'camera' AND stage_id = camRun.cam_id +SET camRun.magicked = 0, + magicDSRun.state = '@NEW_STATE@' +WHERE re_place Index: branches/eam_branches/20090715/ippTools/share/magictool_restore_chip.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magictool_restore_chip.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/magictool_restore_chip.sql (revision 25624) @@ -0,0 +1,8 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN chipRun ON stage = 'chip' AND stage_id = chip_id + JOIN chipProcessedImfile USING(chip_id) +SET chipProcessedImfile.magicked = 0, + chipRun.magicked = 0, + magicDSRun.state = '@NEW_STATE@' +WHERE re_place Index: branches/eam_branches/20090715/ippTools/share/magictool_restore_diff.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magictool_restore_diff.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/magictool_restore_diff.sql (revision 25624) @@ -0,0 +1,8 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN diffRun ON stage = 'diff' AND stage_id = diffRun.diff_id + JOIN diffSkyfile ON diffRun.diff_id = diffSkyfile.diff_id +SET diffRun.magicked = 0, + diffSkyfile.magicked = 0, + magicDSRun.state = '@NEW_STATE@' +WHERE re_place Index: branches/eam_branches/20090715/ippTools/share/magictool_restore_raw.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magictool_restore_raw.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/magictool_restore_raw.sql (revision 25624) @@ -0,0 +1,8 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN rawExp ON stage = 'raw' AND stage_id = rawExp.exp_id + JOIN rawImfile ON rawExp.exp_id = rawImfile.exp_id +SET rawImfile.magicked = 0, + rawExp.magicked = 0, + magicDSRun.state = '@NEW_STATE@' +WHERE re_place Index: branches/eam_branches/20090715/ippTools/share/magictool_restore_warp.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magictool_restore_warp.sql (revision 25624) +++ branches/eam_branches/20090715/ippTools/share/magictool_restore_warp.sql (revision 25624) @@ -0,0 +1,8 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN warpRun ON stage = 'warp' AND stage_id = warp_id + JOIN warpSkyfile using(warp_id) +SET warpRun.magicked = 0, + warpSkyfile.magicked = 0, + magicDSRun.state = '@NEW_STATE@' +WHERE re_place Index: branches/eam_branches/20090715/ippTools/share/pstamptool_revertjob.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/pstamptool_revertjob.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/pstamptool_revertjob.sql (revision 25624) @@ -4,2 +4,4 @@ WHERE pstampRequest.state = 'run' AND pstampJob.state = 'run' + # do not revert faults >= 10 those are faults due to requestor errors + AND pstampJob.fault < 10 Index: branches/eam_branches/20090715/ippTools/share/pxadmin_create_tables.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/pxadmin_create_tables.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/pxadmin_create_tables.sql (revision 25624) @@ -58,4 +58,6 @@ epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, hostname VARCHAR(64), + bytes INT, + md5sum VARCHAR(32), PRIMARY KEY(exp_name, camera, telescope, class, class_id), KEY(fault), @@ -94,4 +96,6 @@ uri VARCHAR(255), epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + bytes INT, + md5sum VARCHAR(32), PRIMARY KEY(exp_id, tmp_class_id), FOREIGN KEY(exp_id) REFERENCES newExp(exp_id) @@ -237,4 +241,7 @@ raw_image_id BIGINT AUTO_INCREMENT, magicked BIGINT, + bytes INT, + md5sum VARCHAR(32), + burntool_state SMALLINT, PRIMARY KEY(exp_id, class_id), KEY(tmp_class_id), @@ -1225,4 +1232,5 @@ name VARCHAR(64) UNIQUE, reqType VARCHAR(16), + label VARCHAR(64), outProduct VARCHAR(64), uri VARCHAR(255), @@ -1263,4 +1271,5 @@ stage VARCHAR(64), stage_id BIGINT, + magic_ds_id BIGINT, label VARCHAR(64), outroot VARCHAR(255), @@ -1290,24 +1299,15 @@ -CREATE TABLE rcDSProduct ( - prod_id BIGINT AUTO_INCREMENT, - name VARCHAR(64), - dbname VARCHAR(64), - dbhost VARCHAR(64), - PRIMARY KEY(prod_id) -) ENGINE=innodb DEFAULT CHARSET=latin1; - CREATE TABLE rcDestination ( dest_id BIGINT AUTO_INCREMENT, - prod_id BIGINT, name VARCHAR(64), status_uri VARCHAR(255), comment VARCHAR(255), last_fileset VARCHAR(255), + dbname VARCHAR(64), + dbhost VARCHAR(64), state VARCHAR(64), - PRIMARY KEY(dest_id), - FOREIGN KEY(prod_id) REFERENCES rcDSProduct(prod_id) + PRIMARY KEY(dest_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; - CREATE TABLE rcInterest ( @@ -1325,14 +1325,13 @@ fs_id BIGINT AUTO_INCREMENT, dist_id BIGINT, - prod_id BIGINT, + dest_id BIGINT, name VARCHAR(255), state VARCHAR(64), registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, fault SMALLINT DEFAULT 0, - PRIMARY KEY(dist_id, prod_id), + PRIMARY KEY(dist_id, dest_id), KEY(fs_id), FOREIGN KEY(dist_id) REFERENCES distRun(dist_id), - FOREIGN KEY(prod_id) REFERENCES rcDSProduct(prod_id) --- KEY(dist_id, prod_id) + FOREIGN KEY(dest_id) REFERENCES rcDestination(dest_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; Index: branches/eam_branches/20090715/ippTools/share/rcserver_updatercrun.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/rcserver_updatercrun.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/rcserver_updatercrun.sql (revision 25624) @@ -1,9 +1,4 @@ UPDATE rcRun JOIN rcDestination USING(dest_id) -JOIN rcDSProduct USING(prod_id) -JOIN rcDSFileset using(prod_id, fs_id) +JOIN rcDSFileset using(dest_id, fs_id) SET -rcRun.fault= 42, rcDestination.last_fileset = 'o4741g0236o.chip.14519.33.1' - - - Index: branches/eam_branches/20090715/ippTools/share/regtool_pendingimfile.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/regtool_pendingimfile.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/regtool_pendingimfile.sql (revision 25624) @@ -6,5 +6,7 @@ newExp.workdir, newImfile.tmp_class_id, - newImfile.uri + newImfile.uri, + newImfile.bytes, + newImfile.md5sum FROM newImfile JOIN newExp Index: branches/eam_branches/20090715/ippTools/share/regtool_updateprocessedimfile.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/regtool_updateprocessedimfile.sql (revision 25623) +++ branches/eam_branches/20090715/ippTools/share/regtool_updateprocessedimfile.sql (revision 25624) @@ -1,4 +1,4 @@ UPDATE rawImfile - SET user_1 = %lf + SET burntool_state = %d WHERE rawImfile.exp_id = %lld Index: branches/eam_branches/20090715/ippTools/src/Makefile.am =================================================================== --- branches/eam_branches/20090715/ippTools/src/Makefile.am (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/Makefile.am (revision 25624) @@ -24,8 +24,4 @@ pubtool - -bin_SCRIPTS = \ - fakemagic - pkginclude_HEADERS = \ pxadd.h \ @@ -36,4 +32,5 @@ pxdata.h \ pxfake.h \ + pxmagic.h \ pxregister.h \ pxtag.h \ @@ -79,4 +76,5 @@ pxfake.c \ pxfault.c \ + pxmagic.c \ pxregister.c \ pxtag.c \ Index: branches/eam_branches/20090715/ippTools/src/difftool.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/difftool.c (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/difftool.c (revision 25624) @@ -37,4 +37,5 @@ static bool todiffskyfileMode(pxConfig *config); static bool adddiffskyfileMode(pxConfig *config); +static bool advanceMode(pxConfig *config); static bool diffskyfileMode(pxConfig *config); static bool revertdiffskyfileMode(pxConfig *config); @@ -50,5 +51,4 @@ static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, psS64 magicked); -static bool diffRunComplete(pxConfig *config); # define MODECASE(caseName, func) \ @@ -76,8 +76,9 @@ MODECASE(DIFFTOOL_MODE_TODIFFSKYFILE, todiffskyfileMode); MODECASE(DIFFTOOL_MODE_ADDDIFFSKYFILE, adddiffskyfileMode); + MODECASE(DIFFTOOL_MODE_ADVANCE, advanceMode); MODECASE(DIFFTOOL_MODE_DIFFSKYFILE, diffskyfileMode); MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileMode); MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunMode); - MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackMode); + MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackMode); MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP, definewarpwarpMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); @@ -555,12 +556,4 @@ } - if (!diffRunComplete(config)) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - // point of no return if (!psDBCommit(config->dbh)) { @@ -569,4 +562,95 @@ } + + return true; +} + +static bool advanceMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + // look for completed diffRuns + psString query = pxDataGet("difftool_completed_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psString whereString = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereString, "\n AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereString); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, query, whereString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(whereString); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + psFree(whereString); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); + psS64 magicked = psMetadataLookupS64(NULL, row, "magicked"); + + // set diffRun.state to 'full' + if (!setdiffRunState(config, diff_id, "full", magicked)) { + psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, + diff_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } return true; @@ -1810,49 +1894,4 @@ } -static bool diffRunComplete(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // look for completed diffRuns - psString query = pxDataGet("difftool_completed_runs.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("difftool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *row = output->data[i]; - - psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); - psS64 magicked = psMetadataLookupS64(NULL, row, "magicked"); - - // set diffRun.state to 'stop' - if (!setdiffRunState(config, diff_id, "full", magicked)) { - psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, - diff_id); - psFree(output); - return false; - } - } - - return true; -} bool exportrunMode(pxConfig *config) Index: branches/eam_branches/20090715/ippTools/src/difftool.h =================================================================== --- branches/eam_branches/20090715/ippTools/src/difftool.h (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/difftool.h (revision 25624) @@ -31,4 +31,5 @@ DIFFTOOL_MODE_TODIFFSKYFILE, DIFFTOOL_MODE_ADDDIFFSKYFILE, + DIFFTOOL_MODE_ADVANCE, DIFFTOOL_MODE_DIFFSKYFILE, DIFFTOOL_MODE_REVERTDIFFSKYFILE, Index: branches/eam_branches/20090715/ippTools/src/difftoolConfig.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/difftoolConfig.c (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/difftoolConfig.c (revision 25624) @@ -114,4 +114,10 @@ psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-good_frac", 0, "define %% of good pixels", NAN); psMetadataAddBool(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked state", false); + + // -advance + psMetadata *advanceArgs = psMetadataAlloc(); + psMetadataAddS64(advanceArgs, PS_LIST_TAIL, "-diff_id", 0, "select by diff ID", 0); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "select by label", NULL); + psMetadataAddS32(advanceArgs, PS_LIST_TAIL, "-limit", 0, "limit number of results", 0); // -diffskyfile @@ -267,4 +273,5 @@ PXOPT_ADD_MODE("-todiffskyfile", "", DIFFTOOL_MODE_TODIFFSKYFILE, todiffskyfileArgs); PXOPT_ADD_MODE("-adddiffskyfile", "", DIFFTOOL_MODE_ADDDIFFSKYFILE, adddiffskyfileArgs); + PXOPT_ADD_MODE("-advance", "", DIFFTOOL_MODE_ADVANCE, advanceArgs); PXOPT_ADD_MODE("-diffskyfile", "", DIFFTOOL_MODE_DIFFSKYFILE, diffskyfileArgs); PXOPT_ADD_MODE("-revertdiffskyfile","", DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileArgs); Index: branches/eam_branches/20090715/ippTools/src/disttool.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/disttool.c (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/disttool.c (revision 25624) @@ -2,5 +2,5 @@ * disttool.c * - * Copyright (C) 2008 + * Copyright (C) 2008-2009 * * This program is free software; you can redistribute it and/or modify it @@ -37,4 +37,5 @@ static bool pendingcomponentMode(pxConfig *config); static bool addprocessedcomponentMode(pxConfig *config); +static bool revertcomponentMode(pxConfig *config); static bool processedcomponentMode(pxConfig *config); static bool toadvanceMode(pxConfig *config); @@ -51,7 +52,4 @@ static bool listtargetMode(pxConfig *config); -static bool definedsproductMode(pxConfig *config); -static bool updatedsproductMode(pxConfig *config); - static bool definedestinationMode(pxConfig *config); static bool updatedestinationMode(pxConfig *config); @@ -59,4 +57,5 @@ static bool defineinterestMode(pxConfig *config); static bool updateinterestMode(pxConfig *config); +static bool listinterestsMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -86,4 +85,5 @@ MODECASE(DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentMode); MODECASE(DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentMode); + MODECASE(DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentMode); MODECASE(DISTTOOL_MODE_TOADVANCE, toadvanceMode); MODECASE(DISTTOOL_MODE_PENDINGFILESET, pendingfilesetMode); @@ -97,10 +97,9 @@ MODECASE(DISTTOOL_MODE_UPDATETARGET, updatetargetMode); MODECASE(DISTTOOL_MODE_LISTTARGET, listtargetMode); - MODECASE(DISTTOOL_MODE_DEFINEDSPRODUCT, definedsproductMode); - MODECASE(DISTTOOL_MODE_UPDATEDSPRODUCT, updatedsproductMode); MODECASE(DISTTOOL_MODE_DEFINEDESTINATION, definedestinationMode); MODECASE(DISTTOOL_MODE_UPDATEDESTINATION, updatedestinationMode); MODECASE(DISTTOOL_MODE_DEFINEINTEREST, defineinterestMode); MODECASE(DISTTOOL_MODE_UPDATEINTEREST, updateinterestMode); + MODECASE(DISTTOOL_MODE_LISTINTERESTS, listinterestsMode); default: psAbort("invalid option (this should not happen)"); @@ -151,4 +150,5 @@ stage, stage_id, + 0, set_label, outroot, @@ -223,5 +223,6 @@ } } else if (!strcmp(stage, "camera")) { - magicRunType = "chipRun"; // This is used below to set the magicked business + magicRunType = "camRun"; // This is used below to set the magicked business + runJoinStr = "camRun.cam_id"; query = pxDataGet("disttool_definebyquery_camera.sql"); if (!query) { @@ -288,5 +289,5 @@ psStringAppend(&query, " AND (stackRun.label = '%s')", label); } - // stack stage doesn't require magic (perhaps let the script do this? + // stack stage doesn't require magic no_magic = true; } else { @@ -306,24 +307,15 @@ if (!no_magic) { - psStringAppend(&query, " AND (distTarget.clean OR %s.magicked)", magicRunType); + psStringAppend(&query, " AND (%s.magicked)", magicRunType); // is selecting by magic_ds_id really interesting? if (magic_ds_id) { - if (strcmp(stage, "camera")) { - // stage other than camera - if (!runJoinStr) { - psError(PS_ERR_PROGRAMMING, true, "cannot select by magic_ds_id for stage: %s", stage); - psFree(query); - return false; - } - psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = distTarget.stage" + if (!runJoinStr) { + psError(PS_ERR_PROGRAMMING, true, "cannot select by magic_ds_id for stage: %s", stage); + psFree(query); + return false; + } + psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = distTarget.stage" " AND magicDSRun.stage_id = %s", runJoinStr); - } else { - // camera masks are magicked when the chipRun is magicked - // XXX: This is confusing. Is it dangerous? - // Maybe I should add a magicked bit to camRun. Note this isn't - psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = 'chip'" - " AND magicDSRun.stage_id = chipRun.chip_id"); - } psStringAppend(&query, " AND (magicDSRun.state = 'full' AND magicDSRun.re_place AND (magic_ds_id = %" PRId64 "))", magic_ds_id); } @@ -377,4 +369,5 @@ psString run_tag = psMetadataLookupStr(NULL, md, "run_tag"); psS64 stage_id = psMetadataLookupS64(NULL, md, "stage_id"); + psS64 magic_ds_id = psMetadataLookupS64(NULL, md, "magicked"); psS64 target_id = psMetadataLookupS64(NULL, md, "target_id"); psString target_label = psMetadataLookupStr(NULL, md, "label"); @@ -395,4 +388,5 @@ stage, stage_id, + magic_ds_id, new_label, outroot, @@ -500,8 +494,5 @@ PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - // we need to disambiguate fault so make a copy of the where list before adding fault - psMetadata *whereComponent = psMetadataCopy(NULL, where); PXOPT_COPY_S16(config->args, where, "-fault", "distRun.fault", "=="); - PXOPT_COPY_S16(config->args, whereComponent, "-fault", "distComponent.fault", "=="); // It might be useful to be able to query by the parameters of the underlying runs @@ -513,84 +504,77 @@ } - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psString query = pxDataGet("disttool_revertrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + int numUpdated = psDBAffectedRows(config->dbh); + + psLogMsg("disttool", PS_LOG_INFO, "Updated %d dist runs", numUpdated); + + return true; +} + +static bool revertcomponentMode(pxConfig *config) +{ + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dist_id", "distRun.dist_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");; + PXOPT_COPY_STR(config->args, where, "-component", "component", "==");; + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + PXOPT_COPY_S16(config->args, where, "-fault", "distComponent.fault", "=="); + + // It might be useful to be able to query by the parameters of the underlying runs + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - return false; - } - - // Update state to 'new' - int numUpdated; // Number updated - { - psString query = pxDataGet("disttool_revertrun_update.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - numUpdated = psDBAffectedRows(config->dbh); - } - - psLogMsg("disttool", PS_LOG_INFO, "Updated %d dist runs", numUpdated); - - // Delete product - int numDeleted; // Number deleted - { - psString query = pxDataGet("disttool_revertrun_delete.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(whereComponent->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(whereComponent, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - numDeleted = psDBAffectedRows(config->dbh); - } + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("disttool_revertcomponent.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + int numDeleted = psDBAffectedRows(config->dbh); psLogMsg("disttool", PS_LOG_INFO, "Deleted %d distComponents", numDeleted); - psFree(where); - psFree(whereComponent); - - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - return true; } @@ -600,16 +584,18 @@ PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); - PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "distRun.label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // look for "inputs" that need to processed - psString query = pxDataGet("disttool_pendingcomponent.sql"); + psString queryFile = NULL; + psStringAppend(&queryFile, "disttool_pending_%s.sql", stage); + psString query = pxDataGet(queryFile); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", queryFile); return false; } @@ -617,5 +603,5 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } @@ -629,28 +615,5 @@ } - // the query has where hooks for each stage. - // right now we aren't using them. - // XXX: I think that I want to change the query from a union of selects on the various - // stages to separate queries. As it is pending data at the later stages of the pipline - // will get blocked by pending earlier stages - psString raw_where = ""; - psString raw_clean_where = ""; - psString chip_where = ""; - psString camera_where = ""; - psString fake_where = ""; - psString warp_where = ""; - psString diff_where = ""; - psString stack_where = ""; - - if (!p_psDBRunQueryF(config->dbh, - query, - raw_where, - raw_clean_where, - chip_where, - camera_where, - fake_where, - warp_where, - diff_where, - stack_where)) { + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -868,4 +831,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "distRun.stage", "=="); pxAddLabelSearchArgs (config, where, "-label", "distRun.label", "=="); @@ -939,5 +903,5 @@ // required values PXOPT_LOOKUP_S64(dist_id, config->args, "-dist_id", true, false); - PXOPT_LOOKUP_S64(prod_id, config->args, "-prod_id", true, false); + PXOPT_LOOKUP_S64(dest_id, config->args, "-dest_id", true, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); @@ -949,5 +913,5 @@ 0, // fs_id dist_id, - prod_id, + dest_id, name, "full", @@ -964,5 +928,5 @@ PXOPT_COPY_S64(config->args, where, "-fs_id", "fs_id", "=="); PXOPT_COPY_S64(config->args, where, "-dist_id", "rcDSFileset.dist_id", "=="); - PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");; PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); @@ -1092,5 +1056,4 @@ PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); - PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "=="); PXOPT_COPY_S64(config->args, where, "-target_id","target_id", "=="); PXOPT_COPY_S64(config->args, where, "-fs_id", "fs_id", "=="); @@ -1433,99 +1396,12 @@ } -static bool definedsproductMode(pxConfig *config) +static bool definedestinationMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); // required - PXOPT_LOOKUP_STR(name, config->args, "-name", true, false); - PXOPT_LOOKUP_STR(dbname, config->args, "-ds_dbname", true, false); - PXOPT_LOOKUP_STR(dbhost, config->args, "-ds_dbhost", true, false); - - // XXX: should we insure that these names do not contatin any whitespace? - - rcDSProductRow *row = rcDSProductRowAlloc( - 0, // prod_id - name, - dbname, - dbhost - ); - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to allocate rcDSProduct object"); - return false; - } - if (!rcDSProductInsertObject(config->dbh, row)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(row); - return false; - } - - // get the assigned target_id - row->prod_id = psDBLastInsertID(config->dbh); - - if (!rcDSProductPrintObject(stdout, row, true)) { - psError(PS_ERR_UNKNOWN, false, "failed to print object"); - psFree(row); - return false; - } - - psFree(row); - - return true; -} -static bool updatedsproductMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "=="); - - PXOPT_LOOKUP_STR(dbname, config->args, "-ds_dbname", false, false); - PXOPT_LOOKUP_STR(dbhost, config->args, "-ds_dbhost", false, false); - - if (!(dbname || dbhost)) { - psError(PS_ERR_UNKNOWN, true, "one or more of dbname or dbhost is required"); - psFree(where); - return false; - } - psString query = psStringCopy("UPDATE rcDSProduct SET"); - psString sep = ""; - if (dbname) { - psStringAppend(&query, " dbname = '%s'", dbname); - sep = ","; - } - if (dbhost) { - psStringAppend(&query, " %s dbhost = '%s'", sep, dbhost); - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); - psFree(whereClause); - } else { - psError(PS_ERR_UNKNOWN, true, "search parameters are required"); - psFree(where); - psFree(query); - return false; - } - psFree(where); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - return true; -} - -static bool definedestinationMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // required - PXOPT_LOOKUP_S64(prod_id, config->args, "-prod_id", true, false); PXOPT_LOOKUP_STR(name, config->args, "-name", true, false); + PXOPT_LOOKUP_STR(dbname, config->args, "-ds_dbname", true, false); + PXOPT_LOOKUP_STR(dbhost, config->args, "-ds_dbhost", true, false); // optional @@ -1539,9 +1415,10 @@ rcDestinationRow *row = rcDestinationRowAlloc( 0, // dest_id - prod_id, name, status_uri, comment, last_fileset, + dbname, + dbhost, state ? state : "enabled" ); @@ -1577,5 +1454,4 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); - PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "=="); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); @@ -1599,5 +1475,5 @@ // last_fileset normally gets set by updatercrunMode // Allowing it to be set here might cause problems - // especially since we are allowing selection by prod_id + // especially since we are allowing selection by dest_id if (last_fileset) { psStringAppend(&query, " %s last_fileset = '%s'", sep, last_fileset); @@ -1631,40 +1507,102 @@ PS_ASSERT_PTR_NON_NULL(config, false); - // required - PXOPT_LOOKUP_S64(dest_id, config->args, "-dest_id", true, false); - PXOPT_LOOKUP_S64(target_id, config->args, "-target_id", true, false); + // one of these is required + PXOPT_LOOKUP_S64(dest_id, config->args, "-dest_id", false, false); + PXOPT_LOOKUP_STR(dest_name, config->args, "-dest_name", false, false); + if (!dest_id && !dest_name) { + psError(PS_ERR_UNKNOWN, true, "either dest_id or dest_name is required"); + return false; + } + + // either target_id or stage and label are required + PXOPT_LOOKUP_S64(target_id, config->args, "-target_id", false, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-filter", false, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + if (!target_id) { + bool error = false; + if (!stage) { + psError(PS_ERR_UNKNOWN, true, "stage is required if target_id is not supplied"); + error = true; + } + if (!label) { + psError(PS_ERR_UNKNOWN, !error, "label is required if target_id is not supplied"); + error = true; + } + if (error) { + return false; + } + } // optional + PXOPT_LOOKUP_S64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - - // XXX: should we insure that these names do not contatin any whitespace? - - rcInterestRow *row = rcInterestRowAlloc( - 0, // int_id - dest_id, - target_id, - state ? state : "enabled" - ); - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to allocate rcInterest object"); - return false; - } - if (!rcInterestInsertObject(config->dbh, row)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(row); - return false; - } - - // get the assigned target_id - row->int_id = psDBLastInsertID(config->dbh); - - if (!rcInterestPrintObject(stdout, row, true)) { - psError(PS_ERR_UNKNOWN, false, "failed to print object"); - psFree(row); - return false; - } - - psFree(row); + if (state) { + if (strcmp(state, "enabled") && strcmp(state, "disabled")) { + psError(PS_ERR_PROGRAMMING, true, "state must be enabled or disabled"); + return false; + } + } else { + // default state + state = "enabled"; + } + + // now that we've done all of our argument checking, copy the values to where + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); + PXOPT_COPY_STR(config->args, where, "-dest_name", "rcDestination.name", "=="); + PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + + psString query = pxDataGet("disttool_defineinterest.sql"); + + if (!psListLength(where->list)) { + // can't get here + psError(PS_ERR_PROGRAMMING, true, "search parameters are required"); + psFree(where); + psFree(query); + return false; + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + psFree(where); + if (clean) { + psStringAppend(&query, " AND (distTarget.clean)"); + } else { + psStringAppend(&query, " AND (!distTarget.clean)"); + } + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + { + // psStringSubstitute fails unless the input is a psString which it determines by + // comparing the memory blocks free function to an expected value. + // pxDataGet uses psSlurp which leaves a different free function on the memory block. + // To work around this make a copy of the query before doing the substitution. + psString queryCopy = psStringCopy(query); + psFree(query); + query = queryCopy; + } + // change the @STATE@ in the sql file to our state + if (!psStringSubstitute(&query, state, "@STATE@")) { + psError(PS_ERR_UNKNOWN, false, "failed to substitute state string"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + int numInserted = psDBAffectedRows(config->dbh); + printf("inserted %d rows into rcInterest\n", numInserted); return true; @@ -1710,3 +1648,83 @@ return true; } - +static bool listinterestsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-int_id", "int_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); + PXOPT_COPY_STR(config->args, where, "-dest_name", "name", "=="); + PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + PXOPT_LOOKUP_BOOL(full, config->args, "-full", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (clean && full) { + psError(PS_ERR_UNKNOWN, false, "can't select both -clean and -full"); + return false; + } + + psString query = pxDataGet("disttool_listinterests.sql"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + if (clean) { + psStringAppend(&query, " AND (clean)"); + } else if (full) { + psStringAppend(&query, " AND (!clean)"); + } + } else if (clean) { + psStringAppend(&query, " WHERE clean"); + } else if (full) { + psStringAppend(&query, " WHERE !clean"); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("disttool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "rcInterest", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} Index: branches/eam_branches/20090715/ippTools/src/disttool.h =================================================================== --- branches/eam_branches/20090715/ippTools/src/disttool.h (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/disttool.h (revision 25624) @@ -31,4 +31,5 @@ DISTTOOL_MODE_PENDINGCOMPONENT, DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, + DISTTOOL_MODE_REVERTCOMPONENT, DISTTOOL_MODE_PROCESSEDCOMPONENT, DISTTOOL_MODE_TOADVANCE, @@ -42,6 +43,4 @@ DISTTOOL_MODE_DEFINEDESTINATION, DISTTOOL_MODE_UPDATEDESTINATION, - DISTTOOL_MODE_DEFINEDSPRODUCT, - DISTTOOL_MODE_UPDATEDSPRODUCT, DISTTOOL_MODE_DEFINETARGET, DISTTOOL_MODE_UPDATETARGET, @@ -49,4 +48,5 @@ DISTTOOL_MODE_DEFINEINTEREST, DISTTOOL_MODE_UPDATEINTEREST, + DISTTOOL_MODE_LISTINTERESTS, } disttoolMode; Index: branches/eam_branches/20090715/ippTools/src/disttoolConfig.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/disttoolConfig.c (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/disttoolConfig.c (revision 25624) @@ -95,6 +95,6 @@ // -pendingcomponent psMetadata *pendingcomponentArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-stage", 0, "limit results to runs for stage (required)", NULL); psMetadataAddS64(pendingcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); - psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-stage", 0, "limit results to runs for stage", NULL); psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit results to label", NULL); psMetadataAddU64(pendingcomponentArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -111,4 +111,15 @@ psMetadataAddS32(addprocessedcomponentArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + // -revertcomponent + psMetadata *revertcomponentArgs = psMetadataAlloc(); + psMetadataAddS64(revertcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); + psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL); + psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); + psMetadataAddS64(revertcomponentArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id", 0); + psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-state", 0, "define state", NULL); + psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddS16(revertcomponentArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + psMetadataAddBool(revertcomponentArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted runs", NULL); + // -processedcomponent psMetadata *processedcomponentArgs = psMetadataAlloc(); @@ -135,5 +146,5 @@ psMetadata *addfilesetArgs = psMetadataAlloc(); psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); - psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id", 0); + psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0); psMetadataAddStr(addfilesetArgs, PS_LIST_TAIL, "-name", 0, "define file name", NULL); psMetadataAddS32(addfilesetArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); @@ -150,5 +161,5 @@ psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-fs_id", 0, "define fs_id", 0); psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); - psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-prod_id", 0, "define dist_id", 0); + psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-dest_id", 0, "define dist_id", 0); psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-stage_id",0, "define stage_id", 0); @@ -162,5 +173,4 @@ psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0); - psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id", 0); psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-fs_id", 0, "define fs_id", 0); @@ -189,22 +199,9 @@ psMetadataAddBool(revertrcrunArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted runs", NULL); - // -definedsproduct - psMetadata *definedsproductArgs = psMetadataAlloc(); - psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-name", 0, "define product name", NULL); - psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name", NULL); - psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host", NULL); - - // -updatedsproduct - // does this mode make sense? - psMetadata *updatedsproductArgs = psMetadataAlloc(); - psMetadataAddS64(updatedsproductArgs, PS_LIST_TAIL, "-prod_id", 0, "select by prod_id", 0); - // can't select by name because it isn't necssarily unique - psMetadataAddStr(updatedsproductArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name", NULL); - psMetadataAddStr(updatedsproductArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host", NULL); - // -definedestination psMetadata *definedestinationArgs = psMetadataAlloc(); - psMetadataAddS64(definedestinationArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id (required)", 0); psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-name", 0, "define destination name (required)", NULL); + psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name (required)", NULL); + psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host (required)", NULL); psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-status_uri", 0, "define status_uri", NULL); psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); @@ -215,10 +212,7 @@ psMetadata *updatedestinationArgs = psMetadataAlloc(); psMetadataAddS64(updatedestinationArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0); - psMetadataAddS64(updatedestinationArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id", 0); psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-name", 0, "define destination name", NULL); psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-status_uri", 0, "define status_uri", NULL); psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); -// last_fileset gets updated by -updatercrun -// psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-set_last_fileset",0, "define last_fileset", NULL); psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); @@ -254,7 +248,13 @@ // -defineinterest psMetadata *defineinterestArgs = psMetadataAlloc(); - psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id (required)", 0); - psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id (required)", 0); + psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-dest_name", 0, "define destination name (LIKE comparison)", NULL); + psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-filter", 0, "define filter (LIKE comparison)", NULL); + psMetadataAddBool(defineinterestArgs, PS_LIST_TAIL,"-clean", 0, "list clean targets", false); psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + psMetadataAddU64(defineinterestArgs, PS_LIST_TAIL, "-limit", 0, "limit number of targets listed to N", 0); // -updateinterest @@ -265,4 +265,19 @@ psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-set_state", 0, "define state (required)", NULL); + // -listinterests + psMetadata *listinterestsArgs = psMetadataAlloc(); + psMetadataAddS64(listinterestsArgs, PS_LIST_TAIL, "-target_id", 0, "list interests with target_id", 0); + psMetadataAddS64(listinterestsArgs, PS_LIST_TAIL, "-dest_id", 0, "list interests with dest_id", 0); + psMetadataAddS64(listinterestsArgs, PS_LIST_TAIL, "-int_id", 0, "list interests with int_id", 0); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-dest_name", 0, "list interests for destinationn name)", NULL); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-label", 0, "list interests for label (LIKE comparison)", NULL); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-filter", 0, "list interests by filter (LIKE comparison)", NULL); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-stage", 0, "list interests for stage", NULL); + psMetadataAddBool(listinterestsArgs, PS_LIST_TAIL,"-clean", 0, "list clean interests", false); + psMetadataAddBool(listinterestsArgs, PS_LIST_TAIL,"-full", 0, "list full interests", false); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-state", 0, "list interests in state", NULL); + psMetadataAddU64(listinterestsArgs, PS_LIST_TAIL, "-limit", 0, "limit number of interests listed to N", 0); + psMetadataAddBool(listinterestsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadata *argSets = psMetadataAlloc(); psMetadata *modes = psMetadataAlloc(); @@ -272,6 +287,7 @@ PXOPT_ADD_MODE("-updaterun", "", DISTTOOL_MODE_UPDATERUN, updaterunArgs); PXOPT_ADD_MODE("-revertrun", "", DISTTOOL_MODE_REVERTRUN, revertrunArgs); - PXOPT_ADD_MODE("-pendingcomponent", "", DISTTOOL_MODE_PENDINGCOMPONENT, pendingcomponentArgs); - PXOPT_ADD_MODE("-addprocessedcomponent", "", DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentArgs); + PXOPT_ADD_MODE("-pendingcomponent", "", DISTTOOL_MODE_PENDINGCOMPONENT, pendingcomponentArgs); + PXOPT_ADD_MODE("-addprocessedcomponent", "", DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentArgs); + PXOPT_ADD_MODE("-revertcomponent", "", DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentArgs); PXOPT_ADD_MODE("-processedcomponent", "", DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentArgs); PXOPT_ADD_MODE("-toadvance", "", DISTTOOL_MODE_TOADVANCE, toadvanceArgs); @@ -284,8 +300,4 @@ PXOPT_ADD_MODE("-pendingdest", "", DISTTOOL_MODE_PENDINGDEST, pendingdestArgs); - PXOPT_ADD_MODE("-definedsproduct", "", DISTTOOL_MODE_DEFINEDSPRODUCT, definedsproductArgs); - PXOPT_ADD_MODE("-updatedsproduct", "", DISTTOOL_MODE_UPDATEDSPRODUCT, updatedsproductArgs); -// PXOPT_ADD_MODE("-listdsproduct", "", DISTTOOL_MODE_LISTDSPRODUCT, updatedsproductArgs); - PXOPT_ADD_MODE("-definedestination", "", DISTTOOL_MODE_DEFINEDESTINATION, definedestinationArgs); PXOPT_ADD_MODE("-updatedestination", "", DISTTOOL_MODE_UPDATEDESTINATION, updatedestinationArgs); @@ -298,5 +310,5 @@ PXOPT_ADD_MODE("-defineinterest", "", DISTTOOL_MODE_DEFINEINTEREST, defineinterestArgs); PXOPT_ADD_MODE("-updateinterest", "", DISTTOOL_MODE_UPDATEINTEREST, updateinterestArgs); -// PXOPT_ADD_MODE("-listinterest", "", DISTTOOL_MODE_LISTINTEREST, listinterestArgs); + PXOPT_ADD_MODE("-listinterests", "", DISTTOOL_MODE_LISTINTERESTS, listinterestsArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/eam_branches/20090715/ippTools/src/magicdstool.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/magicdstool.c (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/magicdstool.c (revision 25624) @@ -37,12 +37,12 @@ static bool todestreakMode(pxConfig *config); static bool adddestreakedfileMode(pxConfig *config); +static bool advancerunMode(pxConfig *config); static bool revertdestreakedfileMode(pxConfig *config); static bool getskycellsMode(pxConfig *config); static bool toremoveMode(pxConfig *config); -static bool torestoreMode(pxConfig *config); static bool torevertMode(pxConfig *config); +static bool completedrevertMode(pxConfig *config); static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, const char *state); -static bool magicDSRunComplete(pxConfig *config, bool setmagicked); static bool magicDSGetIDs(pxConfig *config, psString stage, psS64 magic_id, psS64 *stage_id, psS64 *cam_id); @@ -70,9 +70,10 @@ MODECASE(MAGICDSTOOL_MODE_TODESTREAK, todestreakMode); MODECASE(MAGICDSTOOL_MODE_ADDDESTREAKEDFILE, adddestreakedfileMode); + MODECASE(MAGICDSTOOL_MODE_ADVANCERUN, advancerunMode); MODECASE(MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE,revertdestreakedfileMode); MODECASE(MAGICDSTOOL_MODE_GETSKYCELLS, getskycellsMode); MODECASE(MAGICDSTOOL_MODE_TOREMOVE, toremoveMode); - MODECASE(MAGICDSTOOL_MODE_TORESTORE, torestoreMode); MODECASE(MAGICDSTOOL_MODE_TOREVERT, torevertMode); + MODECASE(MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertMode); default: psAbort("invalid option (this should not happen)"); @@ -105,5 +106,5 @@ PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false); - PXOPT_LOOKUP_BOOL(re_place, config->args, "-replace", false); + PXOPT_LOOKUP_BOOL(noreplace, config->args, "-noreplace", false); PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); @@ -119,5 +120,5 @@ PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); - PXOPT_COPY_S64(config->args, where, "-magic_id","magic_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id","magicRun.magic_id", "=="); PXOPT_COPY_S32(config->args, where, "-streaks_max","streaks", "<="); @@ -268,5 +269,5 @@ outroot, recoveryroot, - re_place, + noreplace ? 0 :1, // re_place 0); // remove @@ -396,23 +397,28 @@ PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // look for "inputs" that need to processed - psString query = pxDataGet("magicdstool_todestreak.sql"); + psString sql_file = NULL; + psStringAppend(&sql_file, "magicdstool_todestreak_%s.sql", stage); + + psString query = pxDataGet(sql_file); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", sql_file); + psFree(sql_file); + return false; + } + psFree(sql_file); if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } @@ -656,13 +662,4 @@ } - if (!magicDSRunComplete(config, setmagicked)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psDBCommit(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -738,7 +735,13 @@ } -static bool magicDSRunComplete(pxConfig *config, bool setmagicked) +static bool advancerunMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); // look for completed magicDSRuns @@ -749,4 +752,10 @@ } + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -766,4 +775,8 @@ return true; } + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } for (long i = 0; i < psArrayLength(output); i++) { psMetadata *row = output->data[i]; @@ -771,8 +784,12 @@ psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id"); - // if requested, set stageRun.magicked + // if re_place, set stageRun.magicked + bool setmagicked = psMetadataLookupBool(NULL, row, "re_place"); if (setmagicked && !setRunMagicked(config, magic_ds_id)) { psError(PS_ERR_UNKNOWN, false, "failed to change stageRun.magicked for magic_ds_id: %" PRId64, magic_ds_id); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -783,8 +800,14 @@ magic_ds_id); psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } } - + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } return true; @@ -795,4 +818,10 @@ { PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(i_am_sure, config->args, "-i_am_sure", true); + if (!i_am_sure) { + psError(PS_ERR_UNKNOWN, true, "Reverting destreaked files must be done carefully. -i_am_sure is required."); + return false; + } psMetadata *where = psMetadataAlloc(); @@ -802,5 +831,5 @@ pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - psString query = psStringCopy("DELETE FROM magicDSFile USING magicDSFile, magicDSRun WHERE (magicDSRun.magic_ds_id = magicDSFile.magic_ds_id) AND magicDSFile.fault != 0"); + psString query = pxDataGet("magicdstool_revertdestreakedfile.sql"); if (psListLength(where->list)) { @@ -808,4 +837,7 @@ psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search arguments are required"); + return false; } psFree(where); @@ -813,40 +845,44 @@ if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "failed to revert"); - return false; - } - return true; -} - -static bool getskycellsMode(pxConfig *config) -{ - // required - PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); + psFree(query); + return false; + } + psFree(query); + return true; +} + +static bool completedrevertMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); - - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psString query = pxDataGet("magicdstool_getskycells.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psString query = pxDataGet("magicdstool_completedrevert.sql"); + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + psString whereString = NULL; if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&whereString, "\nAND %s", whereClause); psFree(whereClause); } psFree(where); - if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } + if (!p_psDBRunQueryF(config->dbh, query, whereString ? whereString : "")) { + psFree(whereString); + psError(PS_ERR_UNKNOWN, false, "failed to revert"); + return false; + } + psFree(whereString); psFree(query); - psArray *output = p_psDBFetchResult(config->dbh); if (!output) { @@ -868,14 +904,24 @@ return true; } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); + for (int i=0; idata[i]; + psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id"); + psString old_state = psMetadataLookupStr(NULL, row, "state"); + psString new_state; + if (!strcmp(old_state, "goto_censored")) { + new_state = "censored"; + } else if (!strcmp(old_state, "goto_restored")) { + new_state = "restored"; + } else { + psError(PXTOOLS_ERR_PROG, true, "unexpected state found: %s", old_state); psFree(output); return false; } - } - + char *query2 = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query2, new_state, magic_ds_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to set run magicDSRun.state to %s", new_state); + return false; + } + } psFree(output); @@ -883,43 +929,16 @@ } -static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!( - (strncmp(state, "new", 4) == 0) - || (strncmp(state, "full", 5) == 0) - ) - ) { - psError(PS_ERR_UNKNOWN, false, - "invalid magicDSRun state: %s", state); - return false; - } - - char *query = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, state, magic_ds_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for magic_id %" PRId64, magic_ds_id); - return false; - } - - return true; -} - -static bool toremoveMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); +static bool getskycellsMode(pxConfig *config) +{ + // required + PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); - PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // look for "inputs" that need to processed - psString query = pxDataGet("magicdstool_toremove.sql"); + psString query = pxDataGet("magicdstool_getskycells.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -929,17 +948,10 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } psFree(where); - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { + if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -970,5 +982,5 @@ if (psArrayLength(output)) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "toremove", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -981,5 +993,35 @@ return true; } -static bool torestoreMode(pxConfig *config) + +static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!((strcmp(state, "new") == 0) || + (strcmp(state, "full") == 0) || + (strcmp(state, "restored") == 0) || + (strcmp(state, "censored") == 0) || + (strcmp(state, "purged") == 0) || + (strcmp(state, "goto_restored") == 0) || + (strcmp(state, "goto_censored") == 0) || + (strcmp(state, "goto_purged") == 0)) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid magicDSRun state: %s", state); + return false; + } + + char *query = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, magic_ds_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for magic_id %" PRId64, magic_ds_id); + return false; + } + + return true; +} + +static bool toremoveMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -994,5 +1036,5 @@ // look for "inputs" that need to processed - psString query = pxDataGet("magicdstool_torestore.sql"); + psString query = pxDataGet("magicdstool_toremove.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -1043,5 +1085,5 @@ if (psArrayLength(output)) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "torestore", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "toremove", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -1054,5 +1096,4 @@ return true; } - static bool torevertMode(pxConfig *config) Index: branches/eam_branches/20090715/ippTools/src/magicdstool.h =================================================================== --- branches/eam_branches/20090715/ippTools/src/magicdstool.h (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/magicdstool.h (revision 25624) @@ -30,9 +30,10 @@ MAGICDSTOOL_MODE_TODESTREAK, MAGICDSTOOL_MODE_ADDDESTREAKEDFILE, + MAGICDSTOOL_MODE_ADVANCERUN, MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE, MAGICDSTOOL_MODE_GETSKYCELLS, MAGICDSTOOL_MODE_TOREMOVE, - MAGICDSTOOL_MODE_TORESTORE, MAGICDSTOOL_MODE_TOREVERT, + MAGICDSTOOL_MODE_COMPLETEDREVERT, } MAGICDStoolMode; Index: branches/eam_branches/20090715/ippTools/src/magicdstoolConfig.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/magicdstoolConfig.c (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/magicdstoolConfig.c (revision 25624) @@ -52,5 +52,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-replace", 0, "replace input files with the destreaked versions", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace input files with the destreaked versions", false); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); @@ -75,6 +75,6 @@ psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-outroot", 0, "define output directory (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-replace", 0, "use the simple output format", false); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-remove", 0, "use the simple output format", false); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace the input wit with the destreaked versions", false); +// psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-remove", 0, "use the simple output format", false); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -93,8 +93,8 @@ // -todestreak psMetadata *todestreakArgs = psMetadataAlloc(); + psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-stage", 0, "limit query to stage (required)", NULL); psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); - psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-stage", 0, "limit query to stage", NULL); psMetadataAddU64(todestreakArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(todestreakArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -114,9 +114,10 @@ psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-component", 0, "search by component", NULL); psMetadataAddS16(revertdestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertdestreakedfileArgs, PS_LIST_TAIL, "-i_am_sure", 0, "confirm that you know what you are doing", false); psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); // -getskycells psMetadata *getskycellsArgs = psMetadataAlloc(); - psMetadataAddS64(getskycellsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic de-streak ID (required)", 0); + psMetadataAddS64(getskycellsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic de-streak ID", 0); psMetadataAddStr(getskycellsArgs, PS_LIST_TAIL, "-class_id", 0, "define class identifier", NULL); psMetadataAddStr(getskycellsArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell identifier", NULL); @@ -130,12 +131,4 @@ psMetadataAddU64(toremoveArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(toremoveArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - - // -torestore - psMetadata *torestoreArgs = psMetadataAlloc(); - psMetadataAddS64(torestoreArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); - psMetadataAddS64(torestoreArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); - psMetadataAddStr(torestoreArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); - psMetadataAddU64(torestoreArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(torestoreArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -torevert @@ -147,4 +140,16 @@ psMetadataAddU64(torevertArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(torevertArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -completedrevert + psMetadata *completedrevertArgs = psMetadataAlloc(); + psMetadataAddS64(completedrevertArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); + psMetadataAddStr(completedrevertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddU64(completedrevertArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -advancerun + psMetadata *advancerunArgs = psMetadataAlloc(); + psMetadataAddS64(advancerunArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); + psMetadataAddStr(advancerunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddU64(advancerunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psFree(now); @@ -163,4 +168,6 @@ PXOPT_ADD_MODE("-adddestreakedfile", "add a de-streaked file", MAGICDSTOOL_MODE_ADDDESTREAKEDFILE, adddestreakedfileArgs); + PXOPT_ADD_MODE("-advancerun", "change state for runs that have finished destrreaking", + MAGICDSTOOL_MODE_ADVANCERUN, advancerunArgs); PXOPT_ADD_MODE("-revertdestreakedfile", " revert a faulted de-streaked file", MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE, revertdestreakedfileArgs); @@ -169,8 +176,10 @@ PXOPT_ADD_MODE("-toremove", "backup images pending removal", MAGICDSTOOL_MODE_TOREMOVE, toremoveArgs); - PXOPT_ADD_MODE("-torestore", "images pending restore of exicsed streak pixels", - MAGICDSTOOL_MODE_TORESTORE, torestoreArgs); - PXOPT_ADD_MODE("-torevert", "faulted images to revert", + PXOPT_ADD_MODE("-torevert", "images to restore or revert", MAGICDSTOOL_MODE_TOREVERT, torevertArgs); + + PXOPT_ADD_MODE("-completedrevert", "change state for runs that have finished reverting", + MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertArgs); + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/eam_branches/20090715/ippTools/src/magictool.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/magictool.c (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/magictool.c (revision 25624) @@ -1393,46 +1393,25 @@ } -static bool censorStage(pxConfig *config, psString stage, psString whereClause) -{ - psString queryFile = NULL; - psStringAppend(&queryFile, "magictool_censor_%s.sql", stage); - psString query = pxDataGet(queryFile); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", queryFile); - psFree(queryFile); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(queryFile); - - psStringAppend(&query, " WHERE %s", whereClause); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - return true; -} - static bool censorrunMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); - psError(PS_ERR_PROGRAMMING, true, "-censorrun mode not ready yet"); - return false; - psMetadata *where = psMetadataAlloc(); + + PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", false, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + + if (!magic_id) { + if (!exp_id && !label) { + psError(PS_ERR_UNKNOWN, true, "either -magic_id or exp_id and label is required"); + return false; + } + } // at least one of these required PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); if (!psListLength(where->list)) { @@ -1470,22 +1449,22 @@ // Now queue any destreaked files to be re-verted - // note: on failure censorStage issues the rollback - if (!censorStage(config, "raw", whereClause)) { + // note: on failure magicRestoreStage issues the rollback + if (!magicRestoreStage(config, "raw", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!censorStage(config, "chip", whereClause)) { + if (!magicRestoreStage(config, "chip", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!censorStage(config, "camera", whereClause)) { + if (!magicRestoreStage(config, "camera", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!censorStage(config, "warp", whereClause)) { + if (!magicRestoreStage(config, "warp", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!censorStage(config, "diff", whereClause)) { + if (!magicRestoreStage(config, "diff", whereClause, "goto_censored")) { psFree(whereClause); return false; Index: branches/eam_branches/20090715/ippTools/src/magictoolConfig.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/magictoolConfig.c (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/magictoolConfig.c (revision 25624) @@ -165,4 +165,5 @@ psMetadataAddS64(censorrunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID", 0); psMetadataAddS64(censorrunArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0); + psMetadataAddStr(censorrunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); psFree(now); Index: branches/eam_branches/20090715/ippTools/src/pstamptool.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/pstamptool.c (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/pstamptool.c (revision 25624) @@ -228,4 +228,5 @@ PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); PXOPT_LOOKUP_S64(ds_id, config->args, "-ds_id", false, false); @@ -236,4 +237,5 @@ name, NULL, // reqType + label, NULL, // outProduct uri, @@ -765,15 +767,19 @@ PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); PXOPT_COPY_S64(config->args, where, "-fault", "fault", "=="); - - if (!psListLength(where->list)) { + PXOPT_COPY_S64(config->args, where, "-req_id_min", "req_id", ">="); + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psString query = pxDataGet("pstamptool_revertjob.sql"); + if (!psListLength(where->list) && !all) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - - psString query = pxDataGet("pstamptool_revertjob.sql"); - psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampJob"); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); + psError(PXTOOLS_ERR_DATA, false, "search parameters or -all are required"); + return false; + } + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampJob"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } psFree(where); Index: branches/eam_branches/20090715/ippTools/src/pstamptoolConfig.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/pstamptoolConfig.c (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/pstamptoolConfig.c (revision 25624) @@ -63,7 +63,8 @@ // -addreq psMetadata *addreqArgs = psMetadataAlloc(); - psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0, "define request file uri (required)", NULL); - psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-ds_id", 0, "define request ds_id", 0); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0, "define request file uri (required)", NULL); + psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-ds_id", 0, "define request ds_id", 0); psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-name", 0, "define request name", NULL); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-label", 0, "define request label", NULL); // -pendingreq @@ -135,6 +136,9 @@ psMetadata *revertjobArgs = psMetadataAlloc(); psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-req_id", 0, "req_id to revert", 0); + psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-req_id_min", 0, "minimum req_id to revert", 0); psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-job_id", 0, "job_id to revert", 0); psMetadataAddS16(revertjobArgs, PS_LIST_TAIL, "-fault", 0, "fault to revert", 0); + psMetadataAddBool(revertjobArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted jobs", false); + psMetadataAddU64(revertjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); // -addproject Index: branches/eam_branches/20090715/ippTools/src/pxinject.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/pxinject.c (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/pxinject.c (revision 25624) @@ -137,7 +137,9 @@ PXOPT_LOOKUP_STR(tmp_class_id, config->args, "-tmp_class_id", true, false); PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false); + PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); // insert with error flag state set to 0 (no errors) - if (!newImfileInsert(config->dbh, exp_id, tmp_class_id, uri, NULL)) { + if (!newImfileInsert(config->dbh, exp_id, tmp_class_id, uri, NULL, bytes, md5sum)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; Index: branches/eam_branches/20090715/ippTools/src/pxinjectConfig.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/pxinjectConfig.c (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/pxinjectConfig.c (revision 25624) @@ -61,4 +61,6 @@ psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-tmp_class_id", 0, "define the class ID (required)", NULL); psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-uri", 0, "define the URI (required)", NULL); + psMetadataAddS32(newImfileArgs, PS_LIST_TAIL, "-bytes", 0, "define the size of the file", 0); + psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-md5sum", 0, "define the size of the file", NULL); // -updatenewExp Index: branches/eam_branches/20090715/ippTools/src/pxmagic.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/pxmagic.c (revision 25624) +++ branches/eam_branches/20090715/ippTools/src/pxmagic.c (revision 25624) @@ -0,0 +1,67 @@ +/* + * pxmagic.c + * + * Copyright (C) 2009 IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxmagic.h" + +bool magicRestoreStage(pxConfig *config, psString stage, psString whereClause, psString newState) +{ + psString queryFile = NULL; + psStringAppend(&queryFile, "magictool_restore_%s.sql", stage); + psString query_temp = pxDataGet(queryFile); + if (!query_temp) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", queryFile); + psFree(queryFile); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(queryFile); + + // psStringSubstittute fails on strings created by pxDataGet() + psString query = psStringCopy(query_temp); + // change the @NEW_STATE@ in the sql file to our state + if (!psStringSubstitute(&query, newState, "@NEW_STATE@")) { + psError(PS_ERR_UNKNOWN, false, "failed to substitute state string"); + return false; + } + + psStringAppend(&query, " AND %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + return true; +} Index: branches/eam_branches/20090715/ippTools/src/pxmagic.h =================================================================== --- branches/eam_branches/20090715/ippTools/src/pxmagic.h (revision 25624) +++ branches/eam_branches/20090715/ippTools/src/pxmagic.h (revision 25624) @@ -0,0 +1,29 @@ +/* + * pxmagic.h + * + * Copyright (C) 2009 IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXMAGIC_H +#define PXMAGIC_H 1 + +#include + +#include "pxtools.h" + +extern bool magicRestoreStage(pxConfig *config, psString stage, psString whereClause, psString newState); + +#endif // PXMAGIC_H Index: branches/eam_branches/20090715/ippTools/src/pxtools.h =================================================================== --- branches/eam_branches/20090715/ippTools/src/pxtools.h (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/pxtools.h (revision 25624) @@ -46,4 +46,5 @@ #include "pxtag.h" #include "pxtree.h" +#include "pxmagic.h" # define MAX_ROWS 10e9 Index: branches/eam_branches/20090715/ippTools/src/pztool.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/pztool.c (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/pztool.c (revision 25624) @@ -401,4 +401,6 @@ PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); PXOPT_LOOKUP_BOOL(row_lock, config->args, "-row_lock", false); + PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false); + PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); // default values @@ -457,5 +459,7 @@ fault, NULL, // epoch - hostname + hostname, + bytes, + md5sum )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -595,5 +599,7 @@ " pzDownloadImfile.class_id," // tmp_class_id " pzDownloadImfile.uri," // uri - " NULL" // epoch + " NULL," // epoch + " pzDownloadImfile.bytes," // bytes + " pzDownloadImfile.md5sum" // md5sum " FROM pzDownloadImfile" " WHERE" Index: branches/eam_branches/20090715/ippTools/src/pztoolConfig.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/pztoolConfig.c (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/pztoolConfig.c (revision 25624) @@ -95,4 +95,6 @@ psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-label", 0, "define the label for the chip stage", NULL); psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-hostname", 0, "define the host that copied the image", NULL); + psMetadataAddS32(copydoneArgs, PS_LIST_TAIL, "-bytes", 0, "define the size in bytes for the copied image", 0); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-md5sum", 0, "define the md5 sum for the copied image", NULL); psMetadataAddS16(copydoneArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); psMetadataAddBool(copydoneArgs, PS_LIST_TAIL, "-row_lock", 0, "lock pzDownImfile rows while advancing an exposure", false); Index: branches/eam_branches/20090715/ippTools/src/regtool.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/regtool.c (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/regtool.c (revision 25624) @@ -224,4 +224,5 @@ PXOPT_LOOKUP_F32(teltemp_extra, config->args, "-teltemp_extra", false, false); PXOPT_LOOKUP_F32(pon_time, config->args, "-pon_time", false, false); + PXOPT_LOOKUP_S16(burntool_state, config->args, "-burntool_state", false, false); PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); @@ -238,4 +239,6 @@ PXOPT_LOOKUP_TIME(dateobs, config->args, "-dateobs", false, false); PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false); + PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); @@ -307,5 +310,8 @@ quality, NULL, - 0 + 0, + bytes, + md5sum, + 0 // burntool_state )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -462,12 +468,13 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); - - if ((fault == INT16_MAX) && !isfinite(user_1)) { - psError(PS_ERR_UNKNOWN, false, "one of -fault or -user_1 must be selected"); - return false; - } - if ((fault != INT16_MAX) && isfinite(user_1)) { - psError(PS_ERR_UNKNOWN, false, "only one of -fault or -user_1 must be selected"); +/* PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); */ + PXOPT_LOOKUP_S16(burntool_state, config->args, "-burntool_state", false, false); + + if ((fault == INT16_MAX) && !isfinite(burntool_state)) { + psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state must be selected"); + return false; + } + if ((fault != INT16_MAX) && isfinite(burntool_state)) { + psError(PS_ERR_UNKNOWN, false, "only one of -fault or -burntool_state must be selected"); return false; } @@ -486,5 +493,5 @@ } - if (isfinite(user_1)) { + if (isfinite(burntool_state)) { psString query = pxDataGet("regtool_updateprocessedimfile.sql"); if (!query) { @@ -493,5 +500,5 @@ } - if (!p_psDBRunQueryF(config->dbh, query, user_1, exp_id, class_id)) { + if (!p_psDBRunQueryF(config->dbh, query, burntool_state, exp_id, class_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); Index: branches/eam_branches/20090715/ippTools/src/regtoolConfig.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/regtoolConfig.c (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/regtoolConfig.c (revision 25624) @@ -125,4 +125,7 @@ ADD_OPT(Time, addprocessedimfileArgs, "-dateobs", "define observation time", NULL); ADD_OPT(Str, addprocessedimfileArgs, "-hostname", "define host name", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-md5sum", "define md5sum", NULL); + ADD_OPT(S32, addprocessedimfileArgs, "-bytes", "define bytes", 0); + ADD_OPT(S16, addprocessedimfileArgs, "-burntool_state", "set burntool state", 0); ADD_OPT(S16, addprocessedimfileArgs, "-fault", "set fault code", 0); ADD_OPT(S16, addprocessedimfileArgs, "-quality", "set quality flag", 0); @@ -153,5 +156,5 @@ ADD_OPT(S64, updateprocessedimfileArgs, "-exp_id", "search by exposure ID", 0); ADD_OPT(Str, updateprocessedimfileArgs, "-class_id", "search by class ID", NULL); - ADD_OPT(F64, updateprocessedimfileArgs, "-user_1", "set user stat (1)", NAN); + ADD_OPT(S16, updateprocessedimfileArgs, "-burntool_state", "set burntool state", 0); ADD_OPT(S16, updateprocessedimfileArgs, "-fault", "set fault code", INT16_MAX); Index: branches/eam_branches/20090715/ippTools/src/stacktool.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/stacktool.c (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/stacktool.c (revision 25624) @@ -139,4 +139,10 @@ PXOPT_COPY_F32(config->args, where, "-select_fwhm_minor_min", "camProcessedExp.fwhm_minor", ">="); PXOPT_COPY_F32(config->args, where, "-select_fwhm_minor_max", "camProcessedExp.fwhm_minor", "<="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m2_max", "camProcessedExp.iq_m2", "<="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m2_min", "camProcessedExp.iq_m2", ">="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m3_max", "camProcessedExp.iq_m3", "<="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m4_min", "camProcessedExp.iq_m4", ">="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m4_max", "camProcessedExp.iq_m4", "<="); + PXOPT_COPY_STR(config->args, where, "-select_exp_type", "rawExp.exp_type", "=="); PXOPT_COPY_F32(config->args, where, "-select_good_frac_min", "warpSkyfile.good_frac", ">="); Index: branches/eam_branches/20090715/ippTools/src/stacktoolConfig.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/stacktoolConfig.c (revision 25623) +++ branches/eam_branches/20090715/ippTools/src/stacktoolConfig.c (revision 25624) @@ -77,4 +77,10 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_fwhm_minor_min", 0, "define min fwhm (minor axis)", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_fwhm_minor_max", 0, "define max fwhm (minor axis)", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m2_min", 0, "define min iq_m2", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m2_max", 0, "define max iq_m2", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m3_min", 0, "define min iq_m3", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m3_max", 0, "define max iq_m3", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m4_min", 0, "define min iq_m4", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m4_max", 0, "define max iq_m4", NAN); psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random", 0, "use this number of random elements", 0); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); Index: branches/eam_branches/20090715/ippconfig/gpc1/camera.config =================================================================== --- branches/eam_branches/20090715/ippconfig/gpc1/camera.config (revision 25623) +++ branches/eam_branches/20090715/ippconfig/gpc1/camera.config (revision 25624) @@ -147,2 +147,4 @@ PHOTCODE.RULE STR {DETECTOR}.{FILTER.ID}.{CHIP.NAME} # Rule for generating photcode + +BURNTOOL.STATE.GOOD S16 13 # Value for burntool_state with the most recent bt version. Index: branches/eam_branches/20090715/ippconfig/gpc1/format_20080925.config =================================================================== --- branches/eam_branches/20090715/ippconfig/gpc1/format_20080925.config (revision 25623) +++ branches/eam_branches/20090715/ippconfig/gpc1/format_20080925.config (revision 25624) @@ -222,4 +222,6 @@ FPA.TELTEMP.TRUSS STR TELTEMTR # Mid truss temperatures (C FPA.TELTEMP.EXTRA STR TELTEMEX # Miscellaneous temperatures (C) + + FPA.BURNTOOL.APPLIED STR BTOOLAPP FPA.PON.TIME STR PONTIME Index: branches/eam_branches/20090715/ippconfig/gpc1/format_20080929.config =================================================================== --- branches/eam_branches/20090715/ippconfig/gpc1/format_20080929.config (revision 25623) +++ branches/eam_branches/20090715/ippconfig/gpc1/format_20080929.config (revision 25624) @@ -217,4 +217,6 @@ FPA.TELTEMP.TRUSS STR TELTEMTR # Mid truss temperatures (C FPA.TELTEMP.EXTRA STR TELTEMEX # Miscellaneous temperatures (C) + + FPA.BURNTOOL.APPLIED STR BTOOLAPP FPA.PON.TIME STR PONTIME Index: branches/eam_branches/20090715/ippconfig/gpc1/format_20081011.config =================================================================== --- branches/eam_branches/20090715/ippconfig/gpc1/format_20081011.config (revision 25623) +++ branches/eam_branches/20090715/ippconfig/gpc1/format_20081011.config (revision 25624) @@ -208,4 +208,6 @@ FPA.TELTEMP.TRUSS STR TELTEMTR # Mid truss temperatures (C FPA.TELTEMP.EXTRA STR TELTEMEX # Miscellaneous temperatures (C) + + FPA.BURNTOOL.APPLIED STR BTOOLAPP FPA.PON.TIME STR PONTIME Index: branches/eam_branches/20090715/ippconfig/gpc1/format_20090120.config =================================================================== --- branches/eam_branches/20090715/ippconfig/gpc1/format_20090120.config (revision 25623) +++ branches/eam_branches/20090715/ippconfig/gpc1/format_20090120.config (revision 25624) @@ -207,4 +207,6 @@ FPA.TELTEMP.TRUSS STR TELTEMTR # Mid truss temperatures (C FPA.TELTEMP.EXTRA STR TELTEMEX # Miscellaneous temperatures (C) + + FPA.BURNTOOL.APPLIED STR BTOOLAPP FPA.PON.TIME STR PONTIME Index: branches/eam_branches/20090715/ippconfig/gpc1/format_20090220.config =================================================================== --- branches/eam_branches/20090715/ippconfig/gpc1/format_20090220.config (revision 25623) +++ branches/eam_branches/20090715/ippconfig/gpc1/format_20090220.config (revision 25624) @@ -207,4 +207,6 @@ FPA.TELTEMP.TRUSS STR TELTEMTR # Mid truss temperatures (C FPA.TELTEMP.EXTRA STR TELTEMEX # Miscellaneous temperatures (C) + + FPA.BURNTOOL.APPLIED STR BTOOLAPP FPA.PON.TIME STR PONTIME Index: branches/eam_branches/20090715/ippconfig/gpc1/format_orig.config =================================================================== --- branches/eam_branches/20090715/ippconfig/gpc1/format_orig.config (revision 25623) +++ branches/eam_branches/20090715/ippconfig/gpc1/format_orig.config (revision 25624) @@ -215,4 +215,6 @@ FPA.TELTEMP.TRUSS STR TELTEMTR # Mid truss temperatures (C FPA.TELTEMP.EXTRA STR TELTEMEX # Miscellaneous temperatures (C) + + FPA.BURNTOOL.APPLIED STR BTOOLAPP FPA.PON.TIME STR PONTIME Index: branches/eam_branches/20090715/ippconfig/recipes/ppImage.config =================================================================== --- branches/eam_branches/20090715/ippconfig/recipes/ppImage.config (revision 25623) +++ branches/eam_branches/20090715/ippconfig/recipes/ppImage.config (revision 25624) @@ -16,5 +16,5 @@ MASK.SATURATED BOOL TRUE # Mask the saturated pixels MASK.LOW BOOL TRUE # Mask pixels below valid range -MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails +MASK.BURNTOOL BOOL FALSE # Mask potential burntool trails VARIANCE.BUILD BOOL FALSE # Build internal variance image PATTERN BOOL FALSE # Fit and remove pattern noise? @@ -26,4 +26,5 @@ CHECK.CTE BOOL FALSE # measure CTE errors? USE.DEBURNED.IMAGE BOOL FALSE # use burntool-repaired image? +USE.BEST.BURNTOOL BOOL TRUE # require the best burntooled data TILTYSTREAK.APPLY BOOL FALSE # apply the 'tiltystreak' tool @@ -56,5 +57,6 @@ SCAN.ROWS S32 100 -BURNTOOL.TRAILS U16 0x01 +# Which regions identified by burntool to mask. 0x01 : only unfit trails, 0x02 "up" trails, 0x04 "down" trails +BURNTOOL.TRAILS U16 0x01 # Non-linearity correction @@ -148,4 +150,5 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails FRINGE BOOL TRUE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -171,4 +174,5 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails FRINGE BOOL TRUE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -194,4 +198,5 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails FRINGE BOOL TRUE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -560,4 +565,5 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails FRINGE BOOL TRUE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -1720,4 +1726,5 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails PATTERN BOOL TRUE # Subtract pattern noise? FRINGE BOOL TRUE # Fringe subtraction Index: branches/eam_branches/20090715/ippconfig/recipes/ppStack.config =================================================================== --- branches/eam_branches/20090715/ippconfig/recipes/ppStack.config (revision 25623) +++ branches/eam_branches/20090715/ippconfig/recipes/ppStack.config (revision 25624) @@ -3,5 +3,5 @@ CONVOLVE BOOL TRUE # Convolve images when stacking? ITER S32 1 # Number of rejection iterations -COMBINE.REJ F32 2.0 # Rejection threshold in combination (sigma) +COMBINE.REJ F32 2.5 # Rejection threshold in combination (sigma) COMBINE.SYS F32 0.03 # Relative systematic error in combination COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean @@ -11,7 +11,7 @@ POOR.FRACTION F32 0.01 # Maximum fraction of bad variance for poor pixels THRESHOLD.MASK F32 0.5 # Threshold for mask deconvolution (0..1) -DECONV.LIMIT F32 5.0 # Deconvolution fraction for rejecting entire image +DECONV.LIMIT F32 1.3 # Deconvolution fraction for rejecting entire image IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image -MATCH.REJ F32 5.0 # Rejection threshold for chi^2 values from matching +MATCH.REJ F32 3.0 # Rejection threshold for chi^2 values from matching ROWS S32 64 # Number of rows to read at once VARIANCE BOOL TRUE # Use variance in rejection? Index: branches/eam_branches/20090715/ippconfig/recipes/ppStats.config =================================================================== --- branches/eam_branches/20090715/ippconfig/recipes/ppStats.config (revision 25623) +++ branches/eam_branches/20090715/ippconfig/recipes/ppStats.config (revision 25624) @@ -190,4 +190,6 @@ CONCEPT STR FPA.TELTEMP.EXTRA # Miscellaneous temperatures (C) + CONCEPT STR FPA.BURNTOOL.APPLIED + CONCEPT STR FPA.PON.TIME # time since last power on Index: branches/eam_branches/20090715/ippconfig/recipes/ppStatsFromMetadata.config =================================================================== --- branches/eam_branches/20090715/ippconfig/recipes/ppStatsFromMetadata.config (revision 25623) +++ branches/eam_branches/20090715/ippconfig/recipes/ppStatsFromMetadata.config (revision 25624) @@ -44,4 +44,5 @@ ENTRY VAL FPA.TELTEMP.EXTRA F32 CONSTANT -teltemp_extra # Miscellaneous temperatures (C) ENTRY VAL FPA.PON.TIME F32 CONSTANT -pon_time # time since last power on +# ENTRY VAL FPA.BURNTOOL.APPLIED S32 CONSTANT -burntool_state # ENTRY VAL CHIP.TEMP F32 SAMPLE_MEAN -ccd_temp # CCD temperature ENTRY VAL CELL.EXPOSURE F32 SAMPLE_MEAN -exp_time # Exposure time Index: branches/eam_branches/20090715/ippconfig/recipes/ppSub.config =================================================================== --- branches/eam_branches/20090715/ippconfig/recipes/ppSub.config (revision 25623) +++ branches/eam_branches/20090715/ippconfig/recipes/ppSub.config (revision 25624) @@ -21,5 +21,5 @@ POOR.FRACTION F32 0.10 # Maximum fraction of bad weight for poor pixels BADFRAC F32 0.95 # Maximum fraction of bad pixels -@ISIS.WIDTHS F32 1 3 5 7 # Gaussian FWHMs for ISIS kernels +@ISIS.WIDTHS F32 3 5 7 10 # Gaussian FWHMs for ISIS kernels @ISIS.ORDERS S32 2 2 2 2 # Polynomial orders for ISIS kernels SPAM.BINNING S32 2 # Binning in outer region for SPAM kernels Index: branches/eam_branches/20090715/ippconfig/recipes/pswarp.config =================================================================== --- branches/eam_branches/20090715/ippconfig/recipes/pswarp.config (revision 25623) +++ branches/eam_branches/20090715/ippconfig/recipes/pswarp.config (revision 25624) @@ -10,6 +10,12 @@ ACCEPT.FRAC F32 0.1 # Minimum fraction of good pixels to accept result INTERPOLATION.NUM S32 1000 # Number of interpolation kernels to pre-calculate +PSF BOOL TRUE # Measure PSF for warped image? # Default recipe for warping WARP METADATA END + +#PR recipe for faster processing +PR METADATA + PSF BOOL FALSE +END Index: branches/eam_branches/20090715/ippconfig/recipes/reductionClasses.mdc =================================================================== --- branches/eam_branches/20090715/ippconfig/recipes/reductionClasses.mdc (revision 25623) +++ branches/eam_branches/20090715/ippconfig/recipes/reductionClasses.mdc (revision 25624) @@ -307,5 +307,5 @@ CHIP_PPIMAGE STR PR CHIP_PSPHOT STR PR - WARP_PSWARP STR WARP + WARP_PSWARP STR PR STACK_PPSTACK STR PR STACK_PPSUB STR PR @@ -324,5 +324,5 @@ CHIP_PPIMAGE STR PR_DARKTEST_NOSUB CHIP_PSPHOT STR PR - WARP_PSWARP STR WARP + WARP_PSWARP STR PR STACK_PPSTACK STR PR STACK_PPSUB STR PR Index: branches/eam_branches/20090715/magic/README =================================================================== --- branches/eam_branches/20090715/magic/README (revision 25623) +++ (revision ) @@ -1,18 +1,0 @@ -To build the DetectStreaks portion of magic - -tar xf ~sydney/magic.tgz -tar xf ~sydney/ssa-core-cpp.tgz - -cd ssa-core-cpp -make distclean -./configure -make - -cd ../magic -make install - -# to build streaksremove and related tools - -cd ../remove/src -make install - Index: branches/eam_branches/20090715/magic/configure.tcsh =================================================================== --- branches/eam_branches/20090715/magic/configure.tcsh (revision 25623) +++ branches/eam_branches/20090715/magic/configure.tcsh (revision 25624) @@ -34,5 +34,4 @@ case --enable-profile case --pedantic - shift breaksw; # key/value options passed by build systems which we ignore Index: branches/eam_branches/20090715/magic/remove/src/streaksio.c =================================================================== --- branches/eam_branches/20090715/magic/remove/src/streaksio.c (revision 25623) +++ branches/eam_branches/20090715/magic/remove/src/streaksio.c (revision 25624) @@ -288,8 +288,8 @@ sfile->resolved_name = psStringCopy(sfile->pmfile->filename); + sfile->nHDU = psFitsGetSize(sfile->pmfile->fits); + // copy header from fpu sfile->header = (psMetadata*) psMemIncrRefCounter(sfile->pmfile->fpa->hdu->header); - - sfile->nHDU = psFitsGetSize(sfile->pmfile->fits); return sfile; Index: branches/eam_branches/20090715/magic/remove/src/streaksremove.c =================================================================== --- branches/eam_branches/20090715/magic/remove/src/streaksremove.c (revision 25623) +++ branches/eam_branches/20090715/magic/remove/src/streaksremove.c (revision 25624) @@ -864,5 +864,10 @@ psF32 y = psMetadataLookupF32(NULL, row, "Y_PSF"); - psImageMaskType mask = maskImage->data.PS_TYPE_IMAGE_MASK_DATA[(int)y][(int)x]; + psImageMaskType mask; + if ((x >= maskImage->numCols) || (y >= maskImage->numRows) || (x < 0) || (y < 0)) { + mask = maskStreak; + } else { + mask = maskImage->data.PS_TYPE_IMAGE_MASK_DATA[(int)y][(int)x]; + } // Key the source if the center pixel is not masked with maskStreak Index: branches/eam_branches/20090715/ppMops/src/ppMopsRead.c =================================================================== --- branches/eam_branches/20090715/ppMops/src/ppMopsRead.c (revision 25623) +++ branches/eam_branches/20090715/ppMops/src/ppMopsRead.c (revision 25624) @@ -98,5 +98,5 @@ det->lengthErr->data.F32[numGood] = 0.0; det->flags->data.U32[numGood] = psMetadataLookupU32(NULL, row, "FLAGS"); - det->diffSkyfileId->data.F32[numGood] = diffSkyfileId; + det->diffSkyfileId->data.S64[numGood] = diffSkyfileId; det->naxis1->data.S32[numGood] = naxis1; det->naxis2->data.S32[numGood] = naxis2; Index: branches/eam_branches/20090715/ppStack/src/ppStackCamera.c =================================================================== --- branches/eam_branches/20090715/ppStack/src/ppStackCamera.c (revision 25623) +++ branches/eam_branches/20090715/ppStack/src/ppStackCamera.c (revision 25624) @@ -85,16 +85,4 @@ psFree(runVars); - psArray *runPSF = pmFPAfileDefineMultipleFromRun(&status, runImages, config, - "PPSTACK.INPUT.PSF"); // Input PSFs - if (!status) { - psError(PS_ERR_UNKNOWN, false, "Unable to define input PSFs from RUN metadata."); - psFree(runImages); - return false; - } - if (runPSF) { - havePSFs = true; - } - psFree(runPSF); - psArray *runSrc = pmFPAfileDefineMultipleFromRun(&status, runImages, config, "PPSTACK.INPUT.SOURCES"); // Input sources @@ -112,18 +100,34 @@ if (convolve) { - psArray *runKernel = pmFPAfileDefineMultipleFromRun(&status, runImages, config, - "PPSTACK.CONV.KERNEL"); // Convolution kernels - if (!status) { - psError(PS_ERR_UNKNOWN, false, "Unable to define convolution kernels from RUN metadata."); - psFree(runImages); - return false; - } - if (!runKernel) { - psError(PS_ERR_UNEXPECTED_NULL, true, - "Unable to define convolution kernels from RUN metadata."); - psFree(runImages); - return false; - } - psFree(runKernel); + { + psArray *runPSF = pmFPAfileDefineMultipleFromRun(&status, runImages, config, + "PPSTACK.INPUT.PSF"); // Input PSFs + if (!status) { + psError(PS_ERR_UNKNOWN, false, "Unable to define input PSFs from RUN metadata."); + psFree(runImages); + return false; + } + if (runPSF) { + havePSFs = true; + } + psFree(runPSF); + } + { + + psArray *runKernel = pmFPAfileDefineMultipleFromRun(&status, runImages, config, + "PPSTACK.CONV.KERNEL"); // Conv'n kernels + if (!status) { + psError(PS_ERR_UNKNOWN, false, "Unable to define convolution kernels from RUN metadata."); + psFree(runImages); + return false; + } + if (!runKernel) { + psError(PS_ERR_UNEXPECTED_NULL, true, + "Unable to define convolution kernels from RUN metadata."); + psFree(runImages); + return false; + } + psFree(runKernel); + } } Index: branches/eam_branches/20090715/ppStack/src/ppStackMatch.c =================================================================== --- branches/eam_branches/20090715/ppStack/src/ppStackMatch.c (revision 25623) +++ branches/eam_branches/20090715/ppStack/src/ppStackMatch.c (revision 25624) @@ -15,5 +15,5 @@ #define SOURCE_MASK (PM_SOURCE_MODE_FAIL | PM_SOURCE_MODE_DEFECT | PM_SOURCE_MODE_SATURATED | \ PM_SOURCE_MODE_CR_LIMIT | PM_SOURCE_MODE_EXT_LIMIT) // Mask to apply to input sources -#define FAINT_SOURCE_FRAC 1.0e-4 // Set minimum flux to this fraction of faintest source flux +#define NOISE_FRACTION 0.01 // Set minimum flux to this fraction of noise #define COVAR_FRAC 0.01 // Truncation fraction for covariance matrix @@ -266,6 +266,8 @@ psRegion *region = psMetadataLookupPtr(NULL, conv->analysis, PM_SUBTRACTION_ANALYSIS_REGION); // Convolution region - - pmSubtractionAnalysis(readout->analysis, kernels, region, + pmSubtractionKernels *kernels = psMetadataLookupPtr(NULL, conv->analysis, + PM_SUBTRACTION_ANALYSIS_KERNEL); + + pmSubtractionAnalysis(readout->analysis, NULL, kernels, region, readout->image->numCols, readout->image->numRows); @@ -317,4 +319,19 @@ pmReadout *fake = pmReadoutAlloc(NULL); // Fake readout with target PSF + psStats *bg = psStatsAlloc(PS_STAT_ROBUST_STDEV); // Statistics for background + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator + if (!psImageBackground(bg, NULL, readout->image, readout->mask, maskVal | maskBad, rng)) { + psError(PS_ERR_UNKNOWN, false, "Can't measure background for image."); + psFree(fake); + psFree(optWidths); + psFree(conv); + psFree(bg); + psFree(rng); + return false; + } + float minFlux = NOISE_FRACTION * bg->robustStdev; // Minimum flux level for fake image + psFree(rng); + psFree(bg); + // For the sake of stamps, remove nearby sources psArray *stampSources = stackSourcesFilter(options->sourceLists->data[index], @@ -323,5 +340,5 @@ if (!pmReadoutFakeFromSources(fake, readout->image->numCols, readout->image->numRows, stampSources, SOURCE_MASK, NULL, NULL, options->psf, - NAN, footprint + size, false, true)) { + minFlux, footprint + size, false, true)) { psError(PS_ERR_UNKNOWN, false, "Unable to generate fake image with target PSF."); psFree(fake); @@ -369,5 +386,5 @@ PM_SUBTRACTION_ANALYSIS_KERNEL); // Conv kernel if (kernel) { - if (!pmSubtractionMatchPrecalc(conv, NULL, readout, fake, readout->analysis, + if (!pmSubtractionMatchPrecalc(NULL, conv, fake, readout, readout->analysis, stride, sysError, maskVal, maskBad, maskPoor, poorFrac, badFrac)) { @@ -383,10 +400,10 @@ } } else { - if (!pmSubtractionMatch(conv, NULL, readout, fake, footprint, stride, regionSize, spacing, + if (!pmSubtractionMatch(NULL, conv, fake, readout, footprint, stride, regionSize, spacing, threshold, stampSources, stampsName, type, size, order, widths, orders, inner, ringsOrder, binning, penalty, optimum, optWidths, optOrder, optThresh, iter, rej, sysError, maskVal, maskBad, maskPoor, poorFrac, badFrac, - PM_SUBTRACTION_MODE_1)) { + PM_SUBTRACTION_MODE_2)) { psError(PS_ERR_UNKNOWN, false, "Unable to match images."); psFree(fake); Index: branches/eam_branches/20090715/ppStack/src/ppStackPSF.c =================================================================== --- branches/eam_branches/20090715/ppStack/src/ppStackPSF.c (revision 25623) +++ branches/eam_branches/20090715/ppStack/src/ppStackPSF.c (revision 25624) @@ -9,7 +9,10 @@ #include "ppStack.h" +//#define TESTING + pmPSF *ppStackPSF(const pmConfig *config, int numCols, int numRows, const psArray *psfs, const psVector *inputMask) { +#ifndef TESTING // Get the recipe values psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe @@ -35,4 +38,12 @@ return NULL; } +#else + // Dummy PSF + pmPSF *psf = pmPSFBuildSimple("PS_MODEL_PS1_V1", 4.0, 4.0, 0.0, 1.0); + if (!psf) { + psError(PS_ERR_UNKNOWN, false, "Unable to build dummy PSF."); + return NULL; + } +#endif return psf; Index: branches/eam_branches/20090715/ppStack/src/ppStackReject.c =================================================================== --- branches/eam_branches/20090715/ppStack/src/ppStackReject.c (revision 25623) +++ branches/eam_branches/20090715/ppStack/src/ppStackReject.c (revision 25624) @@ -9,4 +9,6 @@ #include "ppStack.h" #include "ppStackLoop.h" + +//#define TESTING bool ppStackReject(ppStackOptions *options, pmConfig *config) Index: branches/eam_branches/20090715/psLib/src/fits/psFits.c =================================================================== --- branches/eam_branches/20090715/psLib/src/fits/psFits.c (revision 25623) +++ branches/eam_branches/20090715/psLib/src/fits/psFits.c (revision 25624) @@ -895,4 +895,15 @@ break; + case PS_TYPE_U64: + bitpix = LONGLONG_IMG; + bzero = -1.0 * INT64_MIN; + datatype = TLONGLONG; + break; + + case PS_TYPE_S64: + bitpix = LONGLONG_IMG; + datatype = TLONGLONG; + break; + case PS_TYPE_F32: bitpix = FLOAT_IMG; @@ -921,4 +932,13 @@ _("Specified type, %s, is not supported."), typeStr); + if (bitPix) { + *bitPix = 0; + } + if (dataType) { + *dataType = 0; + } + if (bZero) { + *bZero = 0; + } return false; } @@ -960,2 +980,82 @@ return PS_FITS_COMPRESS_NONE; } + + + +#if 0 +/// Options for FITS I/O +typedef struct { + char *extword; ///< user-specified word to name extensions (NULL implies EXTNAME) + struct { + bool compression; ///< Compression convention: handling of compressed images + bool psBitpix; ///< Custom floating-point image + } conventions; ///< Conventions to honour + // The following options are particular to writing images; they needn't be set for anything else. + psFitsFloat floatType; ///< Desired custom floating-point for output images + int bitpix; ///< Desired BITPIX for output images; 0 to use as provided + psFitsScaling scaling; ///< Scaling scheme to use when quantising floating-point values + bool fuzz; ///< Fuzz the values when quantising floating-point values? + double bscale, bzero; ///< Manually specified BSCALE and BZERO (for SCALE_MANUAL) + double mean, stdev; ///< Mean and standard deviation of image + int stdevBits; ///< Number of bits to sample a standard deviation (for SCALE_STDEV_*) + float stdevNum; ///< Number of standard deviations to pad off the edge +} psFitsOptions; + +bool psFitsCopyCompression(psFits *target, psFits *source) +{ + PS_ASSERT_FITS_NON_NULL(target, false); + PS_ASSERT_FITS_NON_NULL(source, false); + + psMetadata *header = psMetadataReadHeader(NULL, source); // Header of source file + + bool mdok; // Status of MD lookup + psString compTypeStr = psMetadataLookupStr(&mdok, header, "ZCMPTYPE"); // Compression type + psFitsCompressionType compType = psFitsCompressionTypeFromString(compTypeStr); // Compression type + + if (!target->options) { + target->options = psFitsOptionsAlloc(); + } + + target->options->floatType = psFitsFloatImageCheck(source); // Custom floating-point type + + + + + + target->options = psFitsOptionsAlloc(); + target->options->scaling = PS_FITS_SCALE_MANUAL; + target->options->fuzz = false; + target->options->bitpix = bitpix; + target->options->bscale = bscale; + target->options->bzero = bzero; + + psFitsSetCompression(sfile->fits, compType, tiles, 8, 0, 0); + + + + + // Get current BITPIX, BSCALE, BZERO, EXTNAME + // Probably not necessary to look the numerical values up in this + // way, but guards against changes to psLib and cfitsio FITS + // handling. + psMetadataItem *bitpixItem = psMetadataLookup(in->header, "BITPIX"); + psAssert(bitpixItem, "Every FITS image should have BITPIX"); + int bitpix = psMetadataItemParseS32(bitpixItem); + psMetadataItem *bscaleItem = psMetadataLookup(in->header, "BSCALE"); + + float bscale; + if (!bscaleItem) { + psWarning("BSCALE isn't set; defaulting to unity"); + bscale = 1.0; + } else { + bscale = psMetadataItemParseF32(bscaleItem); + } + psMetadataItem *bzeroItem = psMetadataLookup(in->header, "BZERO"); + float bzero; + if (!bzeroItem) { + psWarning("BZERO isn't set; defaulting to zero"); + bzero = 0.0; + } else { + bzero = psMetadataItemParseF32(bzeroItem); + } +#endif Index: branches/eam_branches/20090715/psLib/src/fits/psFitsScale.c =================================================================== --- branches/eam_branches/20090715/psLib/src/fits/psFitsScale.c (revision 25623) +++ branches/eam_branches/20090715/psLib/src/fits/psFitsScale.c (revision 25624) @@ -319,7 +319,7 @@ value = (value - zero) * scale; \ if (options->fuzz) { \ - /* Add random factor [0,1): adds a variance of 1/12, */ \ + /* Add random factor [-0.5,0.5): adds a variance of 1/12, */ \ /* but preserves the expectation value */ \ - value += psRandomUniform(rng); \ + value += psRandomUniform(rng) - 0.5; \ } \ /* Check for underflow and overflow; set either to max */ \ Index: branches/eam_branches/20090715/psconfig/psconfig.bash.in =================================================================== --- branches/eam_branches/20090715/psconfig/psconfig.bash.in (revision 25623) +++ branches/eam_branches/20090715/psconfig/psconfig.bash.in (revision 25624) @@ -35,5 +35,7 @@ # environment variables export PATH=`/bin/csh -f $PSCONFDIR/psconfig.csh --path $version` +export CPATH=`/bin/csh -f $PSCONFDIR/psconfig.csh --cpath $version` export ARCH=`/bin/csh -f $PSCONFDIR/psconfig.csh --arch $version` +export LIBRARY_PATH=`/bin/csh -f $PSCONFDIR/psconfig.csh --library_path $version` export LD_LIBRARY_PATH=`/bin/csh -f $PSCONFDIR/psconfig.csh --ld_library_path $version` export PKG_CONFIG_PATH=`/bin/csh -f $PSCONFDIR/psconfig.csh --pkg_config_path $version` Index: branches/eam_branches/20090715/psconfig/tagsets/ipp-2.9.dist =================================================================== --- branches/eam_branches/20090715/psconfig/tagsets/ipp-2.9.dist (revision 25623) +++ branches/eam_branches/20090715/psconfig/tagsets/ipp-2.9.dist (revision 25624) @@ -74,4 +74,5 @@ YYYYY extsrc/gpcsw ipp-2-9 -0 YYYYY magic ipp-2-9 -0 + YYYYY magic/censorObjects ipp-2-9 -0 # there are externally required C libraries and perl modules (see INSTALL) Index: branches/eam_branches/20090715/psconfig/tagsets/ipp-2.9.libs =================================================================== --- branches/eam_branches/20090715/psconfig/tagsets/ipp-2.9.libs (revision 25623) +++ branches/eam_branches/20090715/psconfig/tagsets/ipp-2.9.libs (revision 25624) @@ -23,5 +23,5 @@ bin autoconf NONE NONE autoconf-2.63.tar.gz autoconf-2.63 N NONE NONE NONE bin automake NONE NONE automake-1.10.tar.gz automake-1.10 N NONE NONE NONE -bin libtool NONE NONE libtool-2.2.6a.tar.gz libtool-2.2.6 N NONE NONE NONE +bin libtool NONE NONE libtool-2.2.6-p1.tar.gz libtool-2.2.6-p1 N NONE NONE NONE bin pkg-config NONE NONE pkg-config-0.23.tar.gz pkg-config-0.23 N NONE NONE NONE Index: branches/eam_branches/20090715/psconfig/tagsets/ipp-2.9.perl =================================================================== --- branches/eam_branches/20090715/psconfig/tagsets/ipp-2.9.perl (revision 25623) +++ branches/eam_branches/20090715/psconfig/tagsets/ipp-2.9.perl (revision 25624) @@ -82,2 +82,7 @@ 78 SQL::Interp SQL-Interp-1.06.tar.gz 79 Log::Dispatch::Email::MailSend Log-Dispatch-2.22.tar.gz + 80 Abstract::Meta::Class Abstract-Meta-Class-0.11.tar.gz + 81 DBIx::Connection DBIx-Connection-0.13.tar.gz + 82 Simple::SAX::Serializer Simple-SAX-Serializer-0.05.tar.gz + 83 Test::Distribution Test-Distribution-2.00.tar.gz + 84 Test::DBUnit Test::DBUnit-0.20.tar.gz Index: branches/eam_branches/20090715/pstamp/scripts/pstamp_finish.pl =================================================================== --- branches/eam_branches/20090715/pstamp/scripts/pstamp_finish.pl (revision 25623) +++ branches/eam_branches/20090715/pstamp/scripts/pstamp_finish.pl (revision 25624) @@ -177,5 +177,5 @@ my $exp_id = $job->{exp_id}; - if ($fault eq $PSTAMP_DUP_REQUEST) { + if (($fault eq $PSTAMP_DUP_REQUEST) and ($req_name eq "NULL")) { # this request had a duplicate request name. We can't put the results # on the data store since the product name is already used @@ -183,5 +183,4 @@ stop_request_and_exit($req_id, $fault); } - my ($row, $req_info, $project) = get_request_info($rows, $rownum); @@ -202,5 +201,5 @@ } - if (($job_type eq "stamp") || ($job_type eq "get_image")) { + if (($job_type eq "stamp") || ($job_type eq "get_image") || ($job_type eq "none")) { my $jreglist = "$out_dir/reglist$job_id"; if (open JRL, "<$jreglist") {; @@ -302,5 +301,10 @@ my $rownum = shift; + if ($rownum eq 0) { + my $dummy_rowinfo = "0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|"; + return (undef, $dummy_rowinfo, "none"); + } my $row = $rows->{$rownum}; + # these may be set to null during processing Index: branches/eam_branches/20090715/pstamp/scripts/pstampparse.pl =================================================================== --- branches/eam_branches/20090715/pstamp/scripts/pstampparse.pl (revision 25623) +++ branches/eam_branches/20090715/pstamp/scripts/pstampparse.pl (revision 25624) @@ -16,4 +16,5 @@ use File::Basename qw(basename); use Carp; +use POSIX qw( strftime ); my $verbose; @@ -98,5 +99,7 @@ my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1"); + # check for duplicate request name +my $duplicate_req_name = 0; if ($req_id and !$no_update) { my $command = "$pstamptool -listreq -name $req_name -not_req_id $req_id"; @@ -109,6 +112,10 @@ if ($success) { # -listreq succeeded duplicate request name + print STDERR "REQ_NAME $req_name has already been used\n"; insertFakeJobForRow(undef, 0, $PSTAMP_DUP_REQUEST); - exit 0; + $duplicate_req_name = 1; + my $datestr = strftime "%Y%m%d%H%M%S.$req_id", gmtime; + $req_name = "ERROR.$datestr"; + #exit 0; } } @@ -126,4 +133,7 @@ my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR); } +} +if ($duplicate_req_name) { + exit 0; } Index: branches/eam_branches/20090715/pstamp/src/ppstampMakeStamp.c =================================================================== --- branches/eam_branches/20090715/pstamp/src/ppstampMakeStamp.c (revision 25623) +++ branches/eam_branches/20090715/pstamp/src/ppstampMakeStamp.c (revision 25624) @@ -100,13 +100,15 @@ // Extract pixels from an image. If part of the bounds of the region are // partially outside of the input those pixels are set to zero. -psImage *extractStampOld(psImage *image, psRegion region) -{ - int width = region.x1 - region.x0; - int height = region.y1 - region.y0; +static psImage *extractStamp(psImage *image, psRegion region, double value) +{ + int width = region.x1 - region.x0 + 0.5; + int height = region.y1 - region.y0 + 0.5; if (width < 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "negative width\n"); return NULL; } if (height < 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "negative height\n"); return NULL; } @@ -132,79 +134,4 @@ int copyWidth = lastX - srcX; - int rightBlank = width - copyWidth - leftBlank; - if (copyWidth <= 0) { - return NULL; - } - - psImage *output = psImageAlloc(width, height, image->type.type); - - psElemType type = image->type.type; - psS32 elementSize = PSELEMTYPE_SIZEOF(type); - int copySize = copyWidth * elementSize; - - for (int dstY=0 ; dstY < height; srcY++, dstY++) { - psU8 *pdst = output->data.U8[dstY]; - - if ((srcY < image->row0) || (srcY >= lastY)) { - // zero the row - memset(pdst, 0, width*elementSize); - } else { - psU8 *psrc = image->data.U8[srcY] + (srcX * elementSize); - - if (leftBlank) { - memset(pdst, 0, leftBlank*elementSize); - } - - // copy copyWidth pixels from srcX,srcY to dstX, dstY - pdst += dstX*elementSize; - memcpy(pdst, psrc, copySize); - - if (rightBlank) { - pdst += copyWidth * elementSize; - memset(pdst, 0, rightBlank); - } - } - } - - - return output; -} -// Extract pixels from an image. If part of the bounds of the region are -// partially outside of the input those pixels are set to zero. -static psImage *extractStamp(psImage *image, psRegion region, double value) -{ - int width = region.x1 - region.x0 + 0.5; - int height = region.y1 - region.y0 + 0.5; - - if (width < 0) { - psError(PS_ERR_BAD_PARAMETER_VALUE, true, "negative width\n"); - return NULL; - } - if (height < 0) { - psError(PS_ERR_BAD_PARAMETER_VALUE, true, "negative height\n"); - return NULL; - } - - int srcX = region.x0; - int srcY = region.y0; - int lastX = region.x1; - int lastY = region.y1; - if (lastX > (image->col0 + image->numCols)) { - lastX = image->col0 + image->numCols; - } - if (lastY > (image->row0 + image->numRows)) { - lastY = image->row0 + image->numRows; - } - - int leftBlank = 0; - int dstX = 0; - if (srcX < image->col0) { - leftBlank = image->col0 - srcX; - dstX += leftBlank; - srcX = 0; - } - - int copyWidth = lastX - srcX; -// int rightBlank = width - copyWidth - leftBlank; if (copyWidth <= 0) { psError(PS_ERR_BAD_PARAMETER_VALUE, true, "copyWidth is invalid: %d\n", copyWidth); @@ -231,5 +158,4 @@ if (skip > 0) { dstY = skip; - height -= skip; srcY = image->row0; } @@ -238,10 +164,9 @@ break; } - psU8 *pdst = output->data.U8[dstY]; + // copy copyWidth pixels from srcX,srcY to dstX, dstY + psU8 *pdst = output->data.U8[dstY] + (dstX * elementSize); psU8 *psrc = image->data.U8[srcY] + (srcX * elementSize); - // copy copyWidth pixels from srcX,srcY to dstX, dstY - pdst += dstX*elementSize; memcpy(pdst, psrc, copySize); } @@ -320,5 +245,5 @@ } - outReadout->image = extractStamp(readout->image, extractRegion, 0); + outReadout->image = extractStamp(readout->image, extractRegion, NAN); if (!outReadout->image) { psError(PS_ERR_UNKNOWN, false, "failed to create postage stamp image\n"); @@ -327,5 +252,5 @@ } if (readout->variance) { - outReadout->variance = extractStamp(readout->variance, extractRegion, 0); + outReadout->variance = extractStamp(readout->variance, extractRegion, NAN); if (!outReadout->variance) { psError(PS_ERR_UNKNOWN, false, "failed to create postage stamp weight image\n"); @@ -396,11 +321,15 @@ -static void skyToChip(pmAstromObj *pt, pmFPA *fpa, pmChip *chip) -{ - // convert from sky to TP to FP - psProject(pt->TP, pt->sky, fpa->toSky); +static void TPToChip(pmAstromObj *pt, pmFPA *fpa, pmChip *chip) +{ psPlaneTransformApply(pt->FP, fpa->fromTPA, pt->TP); // convert from FP to chip psPlaneTransformApply(pt->chip, chip->fromFPA, pt->FP); +} +static void skyToChip(pmAstromObj *pt, pmFPA *fpa, pmChip *chip) +{ + // convert from sky to TP to FP + psProject(pt->TP, pt->sky, fpa->toSky); + TPToChip(pt, fpa, chip); } @@ -412,5 +341,4 @@ psPlaneTransformApply(pt->TP, fpa->toTPA, pt->FP); psDeproject(pt->sky, pt->TP, fpa->toSky); - } @@ -434,5 +362,5 @@ if (!options->roip.celestialCenter) { - // Center was specified in chip coordinates, transform to the sky + // Center was specified in chip coordinates, transform to the sky and the TP chipToSky(center, fpa, chip); } @@ -446,25 +374,28 @@ options->roi.y1 = -INFINITY; - pt->sky->rErr = 0; - pt->sky->dErr = 0; - - pt->sky->r = center->sky->r - options->roip.dRA/2; - pt->sky->d = center->sky->d - options->roip.dDEC/2; - skyToChip(pt, fpa, chip); + double dx = 0.5 * options->roip.dRA / fpa->toSky->Xs; + double dy = 0.5 * options->roip.dDEC / fpa->toSky->Ys; + + pt->TP->xErr = 0; + pt->TP->yErr = 0; + + pt->TP->x = center->TP->x - dx; + pt->TP->y = center->TP->y - dy; + TPToChip(pt, fpa, chip); compareToBox(&options->roi, pt->chip); - pt->sky->r = center->sky->r + options->roip.dRA/2; - pt->sky->d = center->sky->d - options->roip.dDEC/2; - skyToChip(pt, fpa, chip); + pt->TP->x = center->TP->x + dx; + pt->TP->y = center->TP->y - dy; + TPToChip(pt, fpa, chip); compareToBox(&options->roi, pt->chip); - pt->sky->r = center->sky->r + options->roip.dRA/2; - pt->sky->d = center->sky->d + options->roip.dDEC/2; - skyToChip(pt, fpa, chip); + pt->TP->x = center->TP->x + dx; + pt->TP->y = center->TP->y + dy; + TPToChip(pt, fpa, chip); compareToBox(&options->roi, pt->chip); - pt->sky->r = center->sky->r - options->roip.dRA/2; - pt->sky->d = center->sky->d + options->roip.dDEC/2; - skyToChip(pt, fpa, chip); + pt->TP->x = center->TP->x - dx; + pt->TP->y = center->TP->y + dy; + TPToChip(pt, fpa, chip); compareToBox(&options->roi, pt->chip); Index: branches/eam_branches/20090715/pstamp/src/pstamp.h =================================================================== --- branches/eam_branches/20090715/pstamp/src/pstamp.h (revision 25623) +++ branches/eam_branches/20090715/pstamp/src/pstamp.h (revision 25624) @@ -50,4 +50,5 @@ #define PSTAMP_SELECT_MASK 2 #define PSTAMP_SELECT_WEIGHT 4 +#define PSTAMP_SELECT_INVERSE 1024 #define PSTAMP_CENTER_IN_PIXELS 1 @@ -60,5 +61,3 @@ #define STAMP_RESULTS_VERSION "1" - - #endif Index: branches/eam_branches/20090715/pstamp/test/gpc1_sample.txt =================================================================== --- branches/eam_branches/20090715/pstamp/test/gpc1_sample.txt (revision 25623) +++ branches/eam_branches/20090715/pstamp/test/gpc1_sample.txt (revision 25624) @@ -1,24 +1,46 @@ +# Sample Postage stamp request description file +# This can be parsed by the program pstamp_req_create to build a +# PS1_PSTAMP_REQUEST binary table + +# First line of data is for the extension header +# The order of keywords follows TABLE 6 of ICD +# +# Note that value for REQ_NAME may be overriden by a command line +# argument to pstamp_req_create. +# Blank and comment lines are ignored + # REQ_NAME EXTVER -CHANGEME 1 +CHANGEME 1 + +# subsequent lines define the rows in the table + # ROWNUM PROJECT JOB_TYPE OPTION_MASK REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT COORD_MASK CENTER_X CENTER_Y WIDTH HEIGHT REQFILT MJD_MIN MJD_MAX -1 gpc1 stamp 1 byid warp 6245 MD08 skycell.077 2 242.400666 55.273513 200 200 null 0 0 -2 gpc1 stamp 1 byid warp 6254 MD08 skycell.077 2 242.400666 55.273513 200 200 null 0 0 -3 gpc1 stamp 1 byid warp 6264 MD08 skycell.077 2 242.400666 55.273513 200 200 null 0 0 -4 gpc1 stamp 1 byid warp 6317 MD08 skycell.077 2 242.400666 55.273513 200 200 null 0 0 -5 gpc1 stamp 1 byid warp 6324 MD08 skycell.077 2 242.400666 55.273513 200 200 null 0 0 -6 gpc1 stamp 1 byid warp 6465 MD08 skycell.077 2 242.400666 55.273513 200 200 null 0 0 -7 gpc1 stamp 1 byid warp 6466 MD08 skycell.077 2 242.400666 55.273513 200 200 null 0 0 +# warps from various epochs for one of the SN candidates. Specifiying the skycell speeds up processing +1 gpc1 stamp 1 byid warp 6245 null skycell.077 2 242.400666 55.273513 200 200 null 0 0 +2 gpc1 stamp 1 byid warp 6254 null skycell.077 2 242.400666 55.273513 200 200 null 0 0 +3 gpc1 stamp 1 byid warp 6264 null skycell.077 2 242.400666 55.273513 200 200 null 0 0 +4 gpc1 stamp 1 byid warp 6317 null skycell.077 2 242.400666 55.273513 200 200 null 0 0 +5 gpc1 stamp 1 byid warp 6324 null skycell.077 2 242.400666 55.273513 200 200 null 0 0 +6 gpc1 stamp 1 byid warp 6465 null skycell.077 2 242.400666 55.273513 200 200 null 0 0 +7 gpc1 stamp 1 byid warp 6466 null skycell.077 2 242.400666 55.273513 200 200 null 0 0 # get a specific warp -10 gpc1 stamp 1 byid warp 6316 null skycell.077 2 242.400666 55.273513 200 200 null 0 0 +10 gpc1 stamp 1 byid warp 6316 null skycell.077 2 242.400666 55.273513 200 200 null 0 0 + # get stamps from all warps for exposure (only destreaked ones will succeed) -11 gpc1 stamp 1 byexp warp o4973g0123o null skycell.077 2 242.400666 55.273513 200 200 null 0 0 +11 gpc1 stamp 1 byexp warp o4973g0123o null skycell.077 2 242.400666 55.273513 200 200 null 0 0 # get stamps from all chipRuns for exposure (only destreaked ones will succeed) # add the mask and weight images as well -12 gpc1 stamp 7 byexp chip o4973g0123o null all 2 242.400666 55.273513 200 200 null 0 0 +12 gpc1 stamp 7 byexp chip o4973g0123o null null 2 242.400666 55.273513 200 200 null 0 0 # get the corresponding diff -13 gpc1 stamp 1 byexp diff o4973g0123o null null 2 242.400666 55.273513 200 200 null 0 0 +13 gpc1 stamp 1 byexp diff o4973g0123o null null 2 242.400666 55.273513 200 200 null 0 0 +# get the stack that was the templae for that stack +14 gpc1 stamp 1 byid stack 14031 null null 2 242.400666 55.273513 200 200 null 0 0 +# +# get the same stamp by go through the diff +15 gpc1 stamp 1 bydiff stack 193939 null null 2 242.400666 55.273513 200 200 null 0 0 + Index: branches/eam_branches/20090715/pswarp/src/pswarpArguments.c =================================================================== --- branches/eam_branches/20090715/pswarp/src/pswarpArguments.c (revision 25623) +++ branches/eam_branches/20090715/pswarp/src/pswarpArguments.c (revision 25624) @@ -74,9 +74,4 @@ pswarpSetThreads (); - // PSF determination? - if ((N = psArgumentGet(argc, argv, "-psf"))) { - psArgumentRemove(N, &argc, argv); - psMetadataAddBool(config->arguments, PS_LIST_TAIL, "PSF", 0, "Do PSF determination?", true); - } if ((N = psArgumentGet(argc, argv, "-dumpconfig"))) { psArgumentRemove(N, &argc, argv); @@ -163,4 +158,10 @@ } + bool PSF = psMetadataLookupBool(&status, recipe, "PSF"); ///< Generate a PSF model? + if (!status) { + PSF = true; + psWarning("PSF is not set in the %s recipe --- defaulting to TRUE.", PSWARP_RECIPE); + } + // Set recipe values in the recipe (since we've possibly altered some) psMetadataAddS32(recipe, PS_LIST_TAIL, "GRID.NX", PS_META_REPLACE, @@ -174,4 +175,5 @@ psMetadataAddF32(recipe, PS_LIST_TAIL, "POOR.FRAC", PS_META_REPLACE, "Fraction of bad flux for a pixel to be marked as poor", poorFrac); + psMetadataAddBool(recipe, PS_LIST_TAIL, "PSF", PS_META_REPLACE, "Generate a PSF Model?", PSF); // Set recipe values in the arguments @@ -186,4 +188,5 @@ psMetadataAddF32(config->arguments, PS_LIST_TAIL, "POOR.FRAC", 0, "Fraction of bad flux for a pixel to be marked as poor", poorFrac); + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "PSF", PS_META_REPLACE, "Generate a PSF Model?", PSF); psTrace("pswarp", 1, "Done with pswarpArguments...\n"); Index: branches/eam_branches/20090715/pswarp/src/pswarpLoop.c =================================================================== --- branches/eam_branches/20090715/pswarp/src/pswarpLoop.c (revision 25623) +++ branches/eam_branches/20090715/pswarp/src/pswarpLoop.c (revision 25624) @@ -16,5 +16,4 @@ #define WCS_NONLIN_TOL 0.001 // Non-linear tolerance for header WCS -#define PSPHOT_FIND_PSF 1 // Use psphot's findPSF function? #define TESTING 0 // Testing output? @@ -380,5 +379,5 @@ // that's going to be tricky. We have a list of sources, so we use those to redetermine the PSF model. - if (psMetadataLookupBool(&mdok, config->arguments, "PSF")) { + if (psMetadataLookupBool(&mdok, recipe, "PSF")) { fileActivation(config, photFiles, true); ioChecksBefore(config); Index: branches/eam_branches/20090715/pswarp/src/pswarpParseCamera.c =================================================================== --- branches/eam_branches/20090715/pswarp/src/pswarpParseCamera.c (revision 25623) +++ branches/eam_branches/20090715/pswarp/src/pswarpParseCamera.c (revision 25624) @@ -126,6 +126,12 @@ } + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSWARP_RECIPE); + if (!recipe) { + psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSWARP_RECIPE); + return false; + } + bool mdok; // Status of MD lookup - if (psMetadataLookupBool(&mdok, config->arguments, "PSF")) { + if (psMetadataLookupBool(&mdok, recipe, "PSF")) { // This file, PSPHOT.INPUT, is just used as a carrier; output files (eg, PSPHOT.RESID) are defined by // psphotDefineFiles Index: branches/eam_branches/20090715/tools/examine_burntool_pcontrol.pl =================================================================== --- branches/eam_branches/20090715/tools/examine_burntool_pcontrol.pl (revision 25623) +++ branches/eam_branches/20090715/tools/examine_burntool_pcontrol.pl (revision 25624) @@ -2,12 +2,14 @@ # script to print out information about the current status of a burntool run. - use DBI; use Getopt::Std; -getopts('hf',\%opt); +getopts('hfcPv:',\%opt); if (exists($opt{h})) { print STDERR "Usage: examine_burntool_pcontrol.pl [-f] \n"; print STDERR " -f Read all burntool entries, even those commented out.\n"; + print STDERR " -c Print out the needed mysql statement to do a convert. DOES NOT EXECUTE THIS!\n"; + print STDERR " -v Use this value as the target burntool_state in convert statements.\n"; + print STDERR " -P Print out the ipp_apply_burntool.pl commands, even if fully updated.\n"; print STDERR "\n"; print STDERR " Reads in the pcontrol.pro file you're using to run burntool, and uses the\n"; @@ -15,24 +17,52 @@ print STDERR " status of the processing. On the OTA grid:\n"; print STDERR " _ Blank chip\n"; - print STDERR " N Null value for user_1. No burntool attempted.\n"; - print STDERR " O 0.1 value for user_1. Burntool failed or is running.\n"; - print STDERR " B 1.0 value for user_1. Burntool succeeded.\n"; + print STDERR " N Null/zero value for burntool_state. No burntool attempted.\n"; + print STDERR " O -1 value for burntool_state. Burntool is running.\n"; + print STDERR " E -2 value for burntool_state. Error!\n"; + print STDERR " B Current version value for burntool_state. Burntool succeeded.\n"; + print STDERR " b Old version value for burntool_state. Burntool needs to be rerun.\n"; + print STDERR " ? No matching rawImfile entry found in database.\n"; exit(1); } - -# Read a class_id and user_1 pair, and set the correct cell of the (sorry, global) character array vector +$| = 1; +# Determine what the value of "BURNTOOL.STATE.GOOD" currently is: +$burntoolStateGood = 999; +open(LAZY,"ppConfigDump -camera GPC1 -dump-camera - |") || die "Can't run ppConfigDump\n"; +while() { + chomp; + if ($_ =~ /BURNTOOL.STATE.GOOD/) { + @line = split /\s+/; + $burntoolStateGood = $line[2]; + } +} +close(LAZY); +unless( $burntoolStateGood == 999) { + print STDERR "GOOD == $burntoolStateGood\n"; +} +$convert_bt_state = -1.0 * $burntoolStateGood; + +# Read a class_id and burntool_state pair, and set the correct cell of the (sorry, global) character array vector sub class_to_vector { my $class = shift; - my $user1 = shift; + my $burntool_state = shift; my $char = '_'; - if ($user1 == 1.0) { - $char = 'B'; + if (abs($burntool_state) == $burntoolStateGood) { + $char = 'B'; $burncount++; } - elsif ($user1 == 0.1) { + elsif ($burntool_state == -1) { + $char = 'P'; + } + elsif ($burntool_state == 0) { $char = 'O'; } - elsif ($user1 == -1.0) { - $char = 'N'; + elsif ($burntool_state == -2) { + $char = 'E'; + } + elsif ($burntool_state == -3) { + $char = 'X'; + } + else { + $char = 'b'; } @@ -47,5 +77,4 @@ # Engauge autoflush, in case the database server is acting up. $| = 1; - # Load the pcontrol.pro file, and create arrays of the dates included @@ -96,5 +125,6 @@ # Get the data. The limit is there to keep the database happy. I don't think you can observe that many # images on a single night. - $sth = "SELECT exp_id,exp_name,obs_mode,dateobs,class_id,user_1,comment FROM rawImfile WHERE dateobs >= '${dmin}' AND dateobs <= '${dmax}' order by dateobs limit 60000"; + $sth = "SELECT exp_id,exp_name,obs_mode,dateobs,class_id,burntool_state,comment FROM rawImfile WHERE dateobs >= '${dmin}' AND dateobs <= '${dmax}' order by dateobs limit 600000"; + $data_ref = $db->selectall_arrayref( $sth ); @@ -109,4 +139,13 @@ # Print out query for clarity, and the header: print "#### $sth\n"; + if (exists($opt{c})) { + if (exists($opt{v})) { + $no_sth = "UPDATE rawImfile SET burntool_state = $opt{v} WHERE dateobs >= '${dmin}' AND dateobs <= '${dmax}';"; + } + else { + $no_sth = "UPDATE rawImfile SET burntool_state = $convert_bt_state WHERE dateobs >= '${dmin}' AND dateobs <= '${dmax}';"; + } + print "#### $no_sth\n"; + } print "# 0 1 2 3 4 5 6 7 \n"; print "#exp_i exp_name obs_mode dateobs nB 0123456701234567012345670123456701234567012345670123456701234567 comment\n"; @@ -122,19 +161,20 @@ foreach $row_ref (@{ $data_ref }) { - ($exp_id,$exp_name, $obs_mode,$dateobs,$class_id,$user_1,$comment) = @{ $row_ref }; + ($exp_id,$exp_name, $obs_mode,$dateobs,$class_id,$burntool_state,$comment) = @{ $row_ref }; + # Correct for nulls in the database. if (!defined($obs_mode)) { $obs_mode = 'NULL'; } - if (!defined($user_1)) { - $user_1 = -1; + if (!defined($burntool_state)) { + $burntool_state = 0; } if (!defined($comment)) { $comment = ' '; } - +# print "$exp_id $class_id $burntool_state\n"; # If we're still working on the same exp_id, update the chip we got if ($exp_id == $cur_exp_id) { - class_to_vector($class_id,$user_1); + class_to_vector($class_id,$burntool_state); } else { @@ -151,10 +191,10 @@ } else { - $vector[$j] = '?'; + $vector[$j] = '?'; } } } ($cur_exp_id,$cur_exp_name,$cur_obs_mode,$cur_dateobs,$cur_comment) = ($exp_id,$exp_name,$obs_mode,$dateobs,$comment); - class_to_vector($class_id,$user_1); + class_to_vector($class_id,$burntool_state); } } @@ -167,8 +207,8 @@ # If we didn't find that all 60 chips had succesful burntools run, print out the ipp_apply_burntool.pl command we would need to fix this: - if ($burncount != 60) { + if (($burncount != 60)||(exists($opt{P}))) { for ($j = 0; $j < 64; $j++) { - $vector[$j] =~ s/\[..m(.).*/$1/; - if ($vector[$j] ne 'B') { +# printf("%d %s %d\n",$j, $vector[$j], ($vector[$j] !~ /B/)); + if (($vector[$j] !~ /B/)||(exists($opt{P}))) { if ($vector[$j] ne '_') { $id = sprintf("XY%d%d",int($j / 8),$j % 8); Index: branches/eam_branches/20090715/tools/ipp_apply_burntool.pl =================================================================== --- branches/eam_branches/20090715/tools/ipp_apply_burntool.pl (revision 25623) +++ branches/eam_branches/20090715/tools/ipp_apply_burntool.pl (revision 25624) @@ -12,6 +12,6 @@ use Carp; -use IPC::Cmd 0.36 qw( can_run ); -use IPC::Run 0.36 qw( run ); +use IPC::Cmd 0.36 qw( can_run run ); +#use IPC::Run 0.36 qw( run ); use PS::IPP::Metadata::List qw( parse_md_list ); use PS::IPP::Config 1.01 qw( :standard ); @@ -20,15 +20,15 @@ use Pod::Usage qw( pod2usage ); -my ( $class_id, $dateobs_begin, $dateobs_end, $skip_burned, $rerun_from_first, $dbname, $logfile, $verbose, $save_temps); +my ( $class_id, $dateobs_begin, $dateobs_end, $convert, $camera, $dbname, $logfile, $verbose, $save_temps); GetOptions( 'class_id=s' => \$class_id, # chip identifier 'dateobs_begin=s' => \$dateobs_begin, # exposure date/time range start 'dateobs_end=s' => \$dateobs_end, # exposure date/time range stop + 'camera=s' => \$camera, # Camera 'dbname|d=s' => \$dbname, # Database name 'logfile=s' => \$logfile, - 'skip_burned' => \$skip_burned, # Print to stdout - 'rerun_from_first' => \$rerun_from_first, # Print to stdout 'verbose' => \$verbose, # Print to stdout 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); @@ -43,5 +43,8 @@ defined $dbname; -if ($skip_burned and $rerun_from_first) { &my_die("-rerun_from_first and -skip_burned are incompatible"); } +unless (defined $camera) { + $camera = "GPC1"; +# warn("No camera defined. Defaulting to $camera and warning."); +} my $missing_tools; @@ -49,5 +52,6 @@ my $funpack = can_run('funpack') or (warn "Can't find funpack" and $missing_tools = 1); my $burntool = can_run('burntool') or (warn "Can't find burntool" and $missing_tools = 1); -my $fpack = can_run('fpack') or (warn "Can't find fpack" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +#my $fpack = can_run('fpack') or (warn "Can't find fpack" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -57,4 +61,35 @@ my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +# IPP configuration (including nebulous) +my $ipprc = PS::IPP::Config->new( $camera ) or my_die("Unable to set up"); + +$ipprc->redirect_output($logfile) if $logfile; + +# Determine the value of a "good" burntool run. +my $config_cmd = "$ppConfigDump -camera $camera -dump-camera - | grep BURNTOOL | uniq"; +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $config_cmd, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, $class_id, $PS_EXIT_SYS_ERROR); +} + +my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", 0, 0, $class_id, $PS_EXIT_SYS_ERROR); + +my $burntoolStateGood = 999; +foreach my $cfg (@$recipeData) { + if ($cfg->{name} eq 'BURNTOOL.STATE.GOOD') { + $burntoolStateGood = $cfg->{value}; + } +} +if ($burntoolStateGood == 999) { + &my_die("Failed to determine BURNTOOL.STATE.GOOD", $burntoolStateGood, $class_id, 0, $PS_EXIT_SYS_ERROR); +} +my $outState = -1 * abs($burntoolStateGood); + +print ">>$burntoolStateGood<<\n"; + +# Define list of images to examine. my $command = "$regtool -processedimfile"; $command .= " -class_id $class_id"; @@ -64,141 +99,117 @@ $command .= " -dbname $dbname" if defined $dbname; -my @command = split /\s+/, $command; -my ( $stdin, $stdout, $stderr ); # Buffers for running program -print "Running [$command]...\n" if $verbose; -if (not run(\@command, \$stdin, \$stdout, \$stderr)) { - &my_die("Unable to perform regtool -processedimfile"); -} -print $stdout . "\n" if $verbose; - -my @files; -my @whole = split /\n/, $stdout; -my @single = (); -while ( scalar @whole > 0 ) { - my $value = shift @whole; - push @single, $value; - if ($value =~ /^\s*END\s*$/) { - push @single, "\n"; - - my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ); - &my_die("Unable to parse output from regtool") unless defined $list; - push @files, @$list; - @single = (); - } -} - -# IPP configuration (including nebulous) -my $ipprc = PS::IPP::Config->new() or my_die("Unable to set up"); - -$ipprc->redirect_output($logfile) if $logfile; - -my $Nfiles = @files; -print "files: $Nfiles\n"; +( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform regtool: $error_code", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_SYS_ERROR); +} + +my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_SYS_ERROR); + +my $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_SYS_ERROR); my $REALRUN = 1; -my $RAWTABLES = 1; -# XXX if we want to avoid using the raw tables in this script, we need to delay deletion of the tempfile until the next pass. - -# first pass: mark the db entries to catch failures -# these will not be identified as burned by chip processing (user_1 > 0.5) -if (! $skip_burned) { - foreach my $file (@files) { - - # rerun_from_first treats the first image as already burned (if it is already burned) - # the artifact table from the first file is used for the rest of the sequence. - if ($rerun_from_first) { - $skip_burned = 1; - $rerun_from_first = 0; - next; - } - my $exp_id = $file->{exp_id}; - - my $status; - $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -user_1 0.1", 1); + +# Process each file in turn, checking to see if it needs processing, and doing it all in a single pass +my $processNext = 0; +my $previousTable = ''; + +foreach my $file (@$files) { + my $exp_id = $file->{exp_id}; + my $state = $file->{burntool_state}; + + my $rawImfile = $file->{uri}; + my $rawImfileReal = $ipprc->file_resolve($rawImfile, 0); + + my $outTable = $file->{uri}; + $outTable =~ s/fits$/burn.tbl/; + my $outTableReal = $ipprc->file_resolve($outTable, 1); + + my $process = 0; + my $previousTableStyle = 0; + + if ($state == 0) { + $process = 1; + } + elsif ($state == -1) { + $process = 1; + } + elsif ($state == -3) { + $process = 1; + } + elsif ($state == -2) { + &my_die("Aborting as $rawImfile has modified pixel data!"); + } + elsif ($state == $burntoolStateGood) { + $process = 0; + $previousTableStyle = 1; + } + elsif ($state == -1 * $burntoolStateGood) { + $process = 0; +# $previousTable = "in=$outTableReal"; + $previousTableStyle = -1; + } + else { + $process = 1; + } + if ($REALRUN == 0) { + print "##Pretending to process as instructed!\n"; + $process = 1; + } + if (($process == 1)||($processNext == 1)) { + $processNext = 1; + # Set state to processing + my $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -1", $REALRUN); if ($status) { &my_die("failed to update imfile"); } - $file->{user_1} = 0.1; - } -} - -my $prevFileOpt = ""; -foreach my $file (@files) { - my $exp_id = $file->{exp_id}; - - my $rawImfile = $file->{uri}; - my $rawImfileReal; - if ($REALRUN) { - $rawImfileReal = $ipprc->file_resolve($rawImfile, 0); - } else { - ($rawImfileReal) = $rawImfile =~ m|^neb:/(.*)|; - } - print "rawImfile: $rawImfile -> $rawImfileReal\n"; - - # mangle name, create tmp file (always a UNIX file) - my $basename = `basename $rawImfile`; chomp $basename; - my $tempfile = new File::Temp ( TEMPLATE => "$basename.XXXX", - DIR => '/tmp', - UNLINK => !$save_temps, - SUFFIX => '.fits'); - my $tmpImfileReal = $tempfile->filename; - # print "tmpImfile: $tmpImfile -> $tmpImfileReal\n"; - - # destination for the burntool-applied image file (may be a NEB file) - my $outImfile = $rawImfile; - $outImfile =~ s/fits$/burn.fits/; - my $outImfileReal = $ipprc->file_resolve($outImfile, 1); - # print "outImfile: $outImfile -> $outImfileReal\n"; - - # burntool now can write the artifacts to the fits file. - # destination for the burntool artifacts. use this if RAWTABLES is set. - my $artImfile; - my $artImfileReal; - if ($RAWTABLES) { - $artImfile = $rawImfile; - $artImfile =~ s/fits$/burn.tbl/; - $artImfileReal = $ipprc->file_resolve($artImfile, 1); - # print "artImfile: $artImfile -> $artImfileReal\n"; - } - - print "$rawImfile : $skip_burned, $file->{user_1}\n"; - - if (! ($skip_burned and ($file->{user_1} > 0.5))) { - print "running on: $rawImfile\n"; - # uncompress the image (do we need to check if it is compressed?) - my $status = vsystem ("$funpack -S $rawImfileReal > $tmpImfileReal", $REALRUN); - if ($status) { - &my_die("failed on funpack"); - } - - if ($RAWTABLES) { - $status = vsystem ("$burntool $tmpImfileReal out=$artImfileReal $prevFileOpt persist=t", $REALRUN); - } else { - $status = vsystem ("$burntool $tmpImfileReal $prevFileOpt persist=t", $REALRUN); - } - if ($status) { - &my_die("failed on burntool"); - } - - # compress the image (do we need to check if it is compressed?) - $status = vsystem ("$fpack -S $tmpImfileReal > $outImfileReal", $REALRUN); - if ($status) { - &my_die("failed on fpack"); - } - - $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -user_1 1.0", 1); - if ($status) { - &my_die("failed to update imfile"); - } - print "\n"; - } - # save the artifact file for the next image - if ($RAWTABLES) { - $prevFileOpt = "in=$artImfileReal"; - } else { - $prevFileOpt = "infits=$artImfileReal"; - } -} - + $file->{burntool_state} = -1; + + my $tempfile = new File::Temp ( TEMPLATE => "$file->{exp_name}.XXXX", + DIR => '/tmp/', + UNLINK => !$save_temps, + SUFFIX => '.fits'); + my $tempPixels = $tempfile->filename; + + # Run burntool + $status = vsystem ("$funpack -S $rawImfileReal > $tempPixels", $REALRUN); + if ($status) { + &my_die("failed on funpack",$exp_id,$class_id); + } + if ($previousTable ne '') { + $status = vsystem ("$burntool $tempPixels $previousTable out=$outTableReal tableonly=t persist=t", $REALRUN); + } + else { + $status = vsystem ("$burntool $tempPixels out=$outTableReal tableonly=t persist=t", $REALRUN); + } + if ($status) { + &my_die("failed on burntool",$exp_id,$class_id); + } + + $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState", $REALRUN); + if ($status) { + &my_die("failed to update imfile"); + } + + $previousTableStyle = -1; + + print "\n"; + } + + if ($previousTableStyle == 1) { + $previousTable = "infits=$rawImfileReal"; + } + elsif ($previousTableStyle == -1) { + $previousTable = "in=$outTableReal"; + } + else { + die "previousTableStyle undefined!\n"; + } + +} + exit 0; @@ -219,5 +230,9 @@ sub my_die { my $message = shift; - + if ($#_ != -1) { + my $exp_id = shift; + my $class_id = shift; + vsystem("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -3",1); + } printf STDERR "$message\n"; exit 1; Index: branches/eam_branches/20090715/tools/make_burntool_pcontrol.pl =================================================================== --- branches/eam_branches/20090715/tools/make_burntool_pcontrol.pl (revision 25623) +++ branches/eam_branches/20090715/tools/make_burntool_pcontrol.pl (revision 25624) @@ -8,5 +8,5 @@ use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; -getopts('hbQ:d:',\%opt); +getopts('hbPQ:d:',\%opt); if (exists($opt{h})) { print STDERR "Usage: make_burntool_pcontrol.pl -b {-Q | -d | DATE_MIN0 DATE_MAX0 DATE_MIN1 DATE_MAX1 [...]}\n"; @@ -16,4 +16,5 @@ print STDERR " -d DATE Specify a single day to look for.\n"; print STDERR " -b Only print burntool lines.\n"; + print STDERR " -P PR images have bad obs_mode values. Work around that.\n"; print STDERR "\n"; print STDERR " Scans the GPC1 database, and identifies \"science\" observations based on the obs_mode.\n"; @@ -37,5 +38,7 @@ %science = ('MD' => 1, '3PI' => 1, 'STS' => 1, 'CAL' => 1, 'M31' => 1, 'SS' => 1, 'ENGINEERING' => 0, 'NULL' => 0, 'Unknown' => 0, ' ' => 0); - +if (exists($opt{P})) { + $science{Unknown} = 1; +} # Zero the arrays. @dates_min = (); @@ -135,6 +138,15 @@ $science{$obs_mode} = 0; } - - if (($science{$obs_mode} == 1)&&($comment !~ /Daytime/)) { +#%science = ('MD' => 1, '3PI' => 1, 'STS' => 1, 'CAL' => 1, 'M31' => 1, 'SS' => 1, +# 'ENGINEERING' => 0, 'NULL' => 0, 'Unknown' => 0, ' ' => 0); +# I don't like this bit, but we need to work around the bad values of obs_mode that periodically crop up. +# This isn't the most robust solution (which would of course be "having trustworthy obs_mode values"), but +# it'll probably work for now. + if ((($science{$obs_mode} == 1)&&($comment !~ /Daytime/))|| + (($science{$obs_mode} == 0)&&( + ($comment =~ /MD/)||($comment =~ /ThreePi/)|| + ($comment =~ /STS/)||($comment =~ /M31/)|| + ($comment =~ /CAL/)||($comment =~ /SS/)))) + { if ($start_s[-1] == 0) { $start_s[-1] = $et - 1800; Index: branches/eam_branches/20090715/tools/warp_inputs.pl =================================================================== --- branches/eam_branches/20090715/tools/warp_inputs.pl (revision 25623) +++ branches/eam_branches/20090715/tools/warp_inputs.pl (revision 25624) @@ -23,4 +23,5 @@ my ($skycell_id); # Skycell of interest my ($image, $mask, $variance); # Files to contain input lists +my ($muggle); # No magicked inputs? GetOptions( @@ -34,4 +35,5 @@ 'mask=s' => \$mask, # File with masks 'variance=s' => \$variance, # File with variances + 'muggle' => \$muggle, # No magicked inputs? ) or die "Unable to parse arguments.\n"; die "Unknown option: @ARGV\n" if @ARGV; @@ -108,4 +110,11 @@ my $file = shift; # File to copy + if (defined $muggle) { + my @file = split /\//, $file; + my $last = pop @file; + push @file, "SR_$last"; + $file = join '/', @file; + } + my $source = `ipp_datapath.pl $file` or die "Unable to locate $file\n"; # Actual file chomp $source;