Index: branches/eam_branches/20090820/DataStore/Build.PL =================================================================== --- branches/eam_branches/20090820/DataStore/Build.PL (revision 25206) +++ branches/eam_branches/20090820/DataStore/Build.PL (revision 25766) @@ -33,4 +33,5 @@ script_files => [qw( scripts/dsget + scripts/dsgetfileset scripts/dsproductls scripts/dsleech Index: branches/eam_branches/20090820/DataStore/lib/DataStore/Utils.pm =================================================================== --- branches/eam_branches/20090820/DataStore/lib/DataStore/Utils.pm (revision 25206) +++ branches/eam_branches/20090820/DataStore/lib/DataStore/Utils.pm (revision 25766) @@ -42,7 +42,7 @@ $BYTE_FIELD = qr/^\d+$/; $MD5_FIELD = qr/^[0-9a-f]{32}$/; -%KNOWN_FILE_TYPES = map { $_ => 1 } qw( chip psrequest psresults pstamp chipproc warp stack diff ipp-mops table text xml tgz); -%KNOWN_FILESET_TYPES = map { $_ => 1 } qw( OBJECT BIAS DARK SKYFLAT DOMEFLAT OOF SHACKHARTMANN PSREQUEST PSRESULTS IPP-MOPS XRAY FOCUS MOPS_DETECTABILITY_QUERY MOPS_DETECTABILITY_RESPONSE MOPS_TRANSIENT_DETECTIONS LED notset IPP_PSPS); -%KNOWN_PRODUCT_TYPES = map { $_ => 1 } qw( image dump psrequest psresults table ); +%KNOWN_FILE_TYPES = map { $_ => 1 } qw( chip psrequest psresults pstamp chipproc warp stack diff ipp-mops table text xml tgz fits ); +%KNOWN_FILESET_TYPES = map { $_ => 1 } qw( OBJECT BIAS DARK SKYFLAT DOMEFLAT OOF SHACKHARTMANN PSREQUEST PSRESULTS IPP-MOPS XRAY FOCUS MOPS_DETECTABILITY_QUERY MOPS_DETECTABILITY_RESPONSE MOPS_TRANSIENT_DETECTIONS LED notset IPP_PSPS IPP-DIST); +%KNOWN_PRODUCT_TYPES = map { $_ => 1 } qw( image dump psrequest psresults table ipp-dist ipp-misc); =pod Index: branches/eam_branches/20090820/DataStore/scripts/dsget =================================================================== --- branches/eam_branches/20090820/DataStore/scripts/dsget (revision 25206) +++ branches/eam_branches/20090820/DataStore/scripts/dsget (revision 25766) @@ -69,5 +69,5 @@ undef $copies; } - if ($copies < 1) { + elsif ($copies < 1) { die "--copies must be >= 1"; } @@ -207,5 +207,5 @@ if (defined $copies and $copies > 1) { foreach (1 .. ($copies - 1)) { - $neb->replicate($filename, 'any') + $neb->replicate($filename) or die "failed to replicate $filename failed: $!"; } @@ -214,4 +214,7 @@ $tmp->flush or die "can't flush filehandle: $!"; $tmp->sync or die "can't sync filehandle: $!"; + # set read and write permissions based on umask (ignore other bits) + my $umask = umask; + chmod 0666 ^ $umask, $tmpfilename or die "failed to chmod $tmpfilename"; rename $tmpfilename, $filename or die "renaming $tmpfilename to $filename failed: $!"; Index: branches/eam_branches/20090820/DataStore/scripts/dsgetfileset =================================================================== --- branches/eam_branches/20090820/DataStore/scripts/dsgetfileset (revision 25766) +++ branches/eam_branches/20090820/DataStore/scripts/dsgetfileset (revision 25766) @@ -0,0 +1,157 @@ +#!/usr/bin/env perl + +# Copyright (C) 2006-2009 Institute for Astromony University of Hawaii +# +# dsgetfileset + +use strict; +use warnings FATAL => qw( all ); + +use vars qw( $VERSION ); +$VERSION = '0.02'; + +use DataStore; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use File::Basename qw( basename ); + +my ($uri, $outdir, $timeout, $verbose); + +GetOptions( + 'uri|u=s' => \$uri, + 'outdir|o=s' => \$outdir, + 'timeout|t=s' => \$timeout, + 'verbose|v' => \$verbose, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --uri --outdir", + -exitval => 3, +) unless defined $uri and defined $outdir; + +# default http request timeout is 30s +$timeout ||= 30; + +my $response = DataStore::FileSet->new( uri => $uri )->request( + ua_args => { timeout => $timeout }, + ); + +unless (defined $response and $response->is_success) { + warn "request failed: ", $response->status_line; + exit($response->code - 300); +} + +# file retreival succeed + +my $data = $response->data; + +unless ($data) { + warn "no files found"; + exit(0); +} + +print "# uri fileid bytes md5sum type \n"; +foreach my $fs (@$data) { + if ($verbose) { + print + $fs->uri, " ", + $fs->fileid, " ", + $fs->bytes, " ", + $fs->md5sum || 0, " ", + $fs->type; + print " ", join(" ", @{$fs->extra}) if defined $fs->extra; + print "\n"; + } + + my $uri = $fs->uri; + my $base = basename($uri); + my $outfile = "$outdir/$base"; + my $md5 = $fs->md5sum; + my $bytes = $fs->bytes; + + my $command = "dsget --uri $uri --filename $outfile --md5 $md5 --bytes $bytes --timeout $timeout"; + print "$command\n" if $verbose; + my $rc = system $command; + if ($rc) { + my $status = $rc >> 8; + print STDERR "dsget failed: rc: $rc status: $status\n"; + exit $status; + } +} + +__END__ + +=pod + +=head1 NAME + +dsgetfilset - download all the Files under a FileSet from a DataStore server + +=head1 SYNOPSIS + + dsgetfileset --uri --outdir + +=head1 DESCRIPTION + +Download the contents of the Data Store Fileset at the supplied uri to a local directory. + +=head1 OPTIONS + +=over 4 + +=item * --uri|-u + +The URI of the file to be retrieved. + +=item * --outdir|-o + +The output destination directory. + +=item * --timeout|-t + +The amount of time (in seconds) to wait for a response from the DataStore +after making an HTTP request. The default is 30s. + +Optional. + +=back + +=head1 CREDITS + +Based on the DataStore modules and scripts by Joshua Hoblitt + +=head1 SUPPORT + +None. + +=head1 AUTHOR + +The IPP Team + +=head1 COPYRIGHT + +Copyright (C) 2006-2009 Institute for Astromony University of Hawaii + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place - Suite 330, Boston, MA 02111-1307, USA. + +The full text of the license can be found in the L Pod as supplied +with Perl 5.8.1 and later. + +=head1 SEE ALSO + +L, L, L, L, L, L + +=cut Index: branches/eam_branches/20090820/DataStoreServer/scripts/dsfsindex =================================================================== --- branches/eam_branches/20090820/DataStoreServer/scripts/dsfsindex (revision 25206) +++ branches/eam_branches/20090820/DataStoreServer/scripts/dsfsindex (revision 25766) @@ -60,5 +60,5 @@ my $header = "# fileID |bytes |md5sum |type |"; -$stmt = $dbh->prepare("SELECT * from dsFile WHERE fileset_id = $fs_id"); +$stmt = $dbh->prepare("SELECT * from dsFile WHERE fileset_id = $fs_id ORDER BY file_id"); $stmt->execute(); Index: branches/eam_branches/20090820/DataStoreServer/scripts/dsprodtool =================================================================== --- branches/eam_branches/20090820/DataStoreServer/scripts/dsprodtool (revision 25206) +++ branches/eam_branches/20090820/DataStoreServer/scripts/dsprodtool (revision 25766) @@ -12,4 +12,6 @@ use PS::IPP::Metadata::Config; use File::Copy; + +use Term::ReadKey; use PS::IPP::Config qw($PS_EXIT_SUCCESS @@ -36,4 +38,5 @@ my $dsroot; my $dbname; +my $dbpass; my $add; @@ -78,4 +81,23 @@ } else { $product = $del; + if (! -t STDIN) { + die "cannot delete product from script\n"; + } + print "*** Delete the Data Store Product $product? ***\n"; + print "*** to delete $product answer YES, and give password ***\n"; + print "*** WARNING this action is permanant *** \n"; + print "Delete? (YES/[n]): "; + + my $line = ReadLine(0); + chomp $line; + exit 1 if !$line or ($line ne "YES"); + + print "password: "; + ReadMode('noecho'); + $line = ReadLine(0); + ReadMode('normal'); + print "\n"; + chomp $line; + $dbpass = $line; } @@ -93,5 +115,5 @@ } -my $root_index_script = "$dsroot/index.txt"; +my $root_index_script = "$dsroot/index.txt.template"; if (!stat($root_index_script)) { $err .= "Data Store not found at '$dsroot'.\n" @@ -106,11 +128,8 @@ unless defined $dbname; my $dbuser = metadataLookupStr($siteConfig, 'DS_DBUSER'); -my $dbpass = metadataLookupStr($siteConfig, 'DS_DBPASSWORD'); +$dbpass = metadataLookupStr($siteConfig, 'DS_DBPASSWORD') if !$dbpass; exit ($PS_EXIT_CONFIG_ERROR) unless defined $dbserver and $dbname and $dbuser and $dbpass; my $dsn = "DBI:mysql:host=$dbserver;database=$dbname"; - -my $dbh = DBI->connect($dsn, $dbuser, $dbpass) or die "Cannot connect to server\n"; - my $product_dir = "$dsroot/$product"; @@ -118,4 +137,5 @@ if ($del) { + my $dbh = DBI->connect($dsn, $dbuser, $dbpass) or die "Cannot connect to server\n"; # # delete product @@ -129,14 +149,4 @@ } my $prod_id = $prod_row->{prod_id}; - - # if requested, remove the product directory - if ($remove) { - if (system "rm -r $product_dir") { - die("failed to remove $product_dir"); - } - } else { - # otherwise just zap the index script - unlink("$index_script_name"); - } @@ -152,4 +162,17 @@ $dbh->do("DELETE from dsProduct where prod_id = $prod_id"); + print "Product $product deleted.\n"; + + # if requested, remove the product directory + if ($remove) { + print "Removing files from $product.\n"; + if (system "rm -r $product_dir") { + die("failed to remove $product_dir"); + } + } else { + # otherwise just zap the index script + unlink("$index_script_name"); + } + exit 0; @@ -158,4 +181,6 @@ # add a new product # + my $dbh = DBI->connect($dsn, $dbuser, $dbpass) or die "Cannot connect to server\n"; + my $stmt = $dbh->prepare("SELECT prod_id FROM dsProduct WHERE prod_name = \'$product\'"); $stmt->execute(); @@ -167,4 +192,10 @@ # set up the product directory + # if there is an old index file delete it + if (-e $index_script_name ) { + if (!unlink($index_script_name)) { + die("failed trying to remove old $index_script_name"); + } + } if (! -e $product_dir) { $we_created_dir = 1; @@ -172,10 +203,10 @@ die("failed trying to make product directory $product_dir"); } - } - if (-e $index_script_name ) { - if (!unlink($index_script_name)) { - die("failed trying to remove old $index_script_name"); - } - } + } else { + # directory alrady exists make sure that it's empty + my @dirlist = glob("$product_dir/*"); + die ("existing product directory $product_dir is not empty") if scalar @dirlist; + } + if (!copy($root_index_script, $index_script_name)) { print STDERR "failed trying to copy($root_index_script, $index_script_name)"; Index: branches/eam_branches/20090820/DataStoreServer/scripts/dsreg =================================================================== --- branches/eam_branches/20090820/DataStoreServer/scripts/dsreg (revision 25206) +++ branches/eam_branches/20090820/DataStoreServer/scripts/dsreg (revision 25766) @@ -203,8 +203,8 @@ $new_last_fs = $new_newest->{fileset_name}; } else { - $new_last_fs = "none"; + $new_last_fs = undef; } $stmt->finish(); - $dbh->do("UPDATE dsProduct SET last_fs = \'$new_last_fs\' WHERE prod_id = $prod_id"); + $dbh->do("UPDATE dsProduct SET last_fs = ? WHERE prod_id = $prod_id", undef, ($new_last_fs)); } $dbh->commit(); Index: branches/eam_branches/20090820/DataStoreServer/scripts/installscripts =================================================================== --- branches/eam_branches/20090820/DataStoreServer/scripts/installscripts (revision 25206) +++ branches/eam_branches/20090820/DataStoreServer/scripts/installscripts (revision 25766) @@ -56,5 +56,5 @@ # index.txt file for the root of the data store. my @dsroot_list = qw( -index.txt +index.txt.template ); Index: branches/eam_branches/20090820/DataStoreServer/scripts/tabledefs.sql =================================================================== --- branches/eam_branches/20090820/DataStoreServer/scripts/tabledefs.sql (revision 25206) +++ branches/eam_branches/20090820/DataStoreServer/scripts/tabledefs.sql (revision 25766) @@ -75,4 +75,7 @@ # type specific column for type chip INSERT INTO dsFileType (type, type_col_0) VALUES('chip', 'chipname'); +INSERT INTO dsFileType (type, type_col_0) VALUES('dbinfo', 'component'); +INSERT INTO dsFileType (type, type_col_0) VALUES('text', 'component'); +INSERT INTO dsFileType (type, type_col_0) VALUES('tgz', 'component'); # none of these types have any type specific columns Index: branches/eam_branches/20090820/DataStoreServer/web/cgi/dsgetindex =================================================================== --- branches/eam_branches/20090820/DataStoreServer/web/cgi/dsgetindex (revision 25206) +++ branches/eam_branches/20090820/DataStoreServer/web/cgi/dsgetindex (revision 25766) @@ -140,7 +140,9 @@ # 'text/plain' format print header('text/plain', '404 ERROR'); - #print header('text/plain', '200 OK'); +# print header('text/plain', '410 GONE'); +# print header('text/plain', '200 OK'); print @$stderr_buf; print STDERR @$stderr_buf; +# exit 410; } } @@ -172,17 +174,26 @@ # -sub print_free_space { - use Filesys::Df; - # get free space - my $ref = df($DS_ROOT); - printf '

Free space: %.2f G (%.1f%%)

', - $ref->{bfree}/(1024*1024), - 100.0*$ref->{bfree}/$ref->{blocks}; - print "\n"; -} +#sub print_free_space { +# use Filesys::Df; +# # get free space +# my $ref = df($DS_ROOT); +# printf '

Free space: %.2f G (%.1f%%)

', +# $ref->{bfree}/(1024*1024), +# 100.0*$ref->{bfree}/$ref->{blocks}; +# print "\n"; +#} sub pprint_pre { print header('text/html', '200 OK'); - print start_html('Data Store'); + my $name; + # note $fileset is not so we look at the name of the index script to determine the level + if ($program eq "dsfsindex") { + $name = $fileset; + } elsif ($program eq "dsprodindex") { + $name = "$product"; + } else { + $name = ""; + } + print start_html("IPP Data Store $name"); @@ -208,5 +219,5 @@ print p(); - print_free_space(); +# print_free_space(); print end_html(); } @@ -241,8 +252,10 @@ # if this is a file request, use a download cgi if ($wpath =~ /\.fits\s*$/) { - $wpath = '/ds-cgi/dsfits.cgi?'.substr($wpath, 4); # strip' /ds/' + # This doesn't work through the proxy + # The /ds-cgi gets tried on alala + # $wpath = '/ds-cgi/dsfits.cgi?'.substr($wpath, 4); # strip' /ds/' } - print a({-href=>$wpath}, $toks[0]); + print a({-href=>$wpath, -type=>'image/x-fits'}, $toks[0]); } Index: branches/eam_branches/20090820/DataStoreServer/web/cgi/index.txt =================================================================== --- branches/eam_branches/20090820/DataStoreServer/web/cgi/index.txt (revision 25206) +++ (revision ) @@ -1,1 +1,0 @@ - Index: branches/eam_branches/20090820/DataStoreServer/web/cgi/index.txt.template =================================================================== --- branches/eam_branches/20090820/DataStoreServer/web/cgi/index.txt.template (revision 25766) +++ branches/eam_branches/20090820/DataStoreServer/web/cgi/index.txt.template (revision 25766) @@ -0,0 +1,1 @@ + Index: branches/eam_branches/20090820/Ohana/src/addstar/src/ConfigInit.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/addstar/src/ConfigInit.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/addstar/src/ConfigInit.c (revision 25766) @@ -180,5 +180,5 @@ /* XXX this does not yet write out the master photcode table */ sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile)) { + if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) { fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); exit (1); Index: branches/eam_branches/20090820/Ohana/src/addstar/src/ConfigInit_skycells.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/addstar/src/ConfigInit_skycells.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/addstar/src/ConfigInit_skycells.c (revision 25766) @@ -55,5 +55,5 @@ /* XXX this does not yet write out the master photcode table */ sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile)) { + if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) { fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); exit (1); Index: branches/eam_branches/20090820/Ohana/src/addstar/src/addstar.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/addstar/src/addstar.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/addstar/src/addstar.c (revision 25766) @@ -25,5 +25,5 @@ options = args (argc, argv, options); - sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, SKY_DEPTH, VERBOSE); + sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); if (sky == NULL) { fprintf (stderr, "ERROR: unable to load sky table data\n"); Index: branches/eam_branches/20090820/Ohana/src/addstar/src/addstard.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/addstar/src/addstard.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/addstar/src/addstard.c (revision 25766) @@ -12,5 +12,5 @@ /* store the sky table in a global for internal use */ - ServerSky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, SKY_DEPTH, VERBOSE); + ServerSky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); SkyTableSetFilenames (ServerSky, CATDIR, "cpt"); Index: branches/eam_branches/20090820/Ohana/src/addstar/src/addstart.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/addstar/src/addstart.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/addstar/src/addstart.c (revision 25766) @@ -12,5 +12,5 @@ /* store the sky table in a global for internal use */ - ServerSky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, SKY_DEPTH, VERBOSE); + ServerSky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); SkyTableSetFilenames (ServerSky, CATDIR, "cpt"); Index: branches/eam_branches/20090820/Ohana/src/addstar/src/load2mass.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/addstar/src/load2mass.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/addstar/src/load2mass.c (revision 25766) @@ -16,5 +16,5 @@ // load the full sky description table: - sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, SKY_DEPTH, VERBOSE); + sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); SkyTableSetFilenames (sky, CATDIR, "cpt"); Index: branches/eam_branches/20090820/Ohana/src/addstar/src/sedstar.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/addstar/src/sedstar.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/addstar/src/sedstar.c (revision 25766) @@ -15,5 +15,5 @@ options = args_sedstar (argc, argv, options); - sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, SKY_DEPTH, VERBOSE); + sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); SkyTableSetFilenames (sky, CATDIR, "cpt"); Index: branches/eam_branches/20090820/Ohana/src/delstar/src/ConfigInit.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/delstar/src/ConfigInit.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/delstar/src/ConfigInit.c (revision 25766) @@ -45,5 +45,5 @@ /* XXX this does not yet write out the master photcode table */ sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile)) { + if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) { fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); exit (1); Index: branches/eam_branches/20090820/Ohana/src/delstar/src/delete_imagefile.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/delstar/src/delete_imagefile.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/delstar/src/delete_imagefile.c (revision 25766) @@ -12,5 +12,5 @@ /* load sky from correct table */ - sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, SKY_DEPTH, VERBOSE); + sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); SkyTableSetFilenames (sky, CATDIR, "cpt"); Index: branches/eam_branches/20090820/Ohana/src/delstar/src/delete_imagename.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/delstar/src/delete_imagename.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/delstar/src/delete_imagename.c (revision 25766) @@ -15,5 +15,5 @@ /* load sky from correct table */ - sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, SKY_DEPTH, VERBOSE); + sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); SkyTableSetFilenames (sky, CATDIR, "cpt"); Index: branches/eam_branches/20090820/Ohana/src/delstar/src/delete_times.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/delstar/src/delete_times.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/delstar/src/delete_times.c (revision 25766) @@ -15,5 +15,5 @@ /* load sky from correct table */ - sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, SKY_DEPTH, VERBOSE); + sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); SkyTableSetFilenames (sky, CATDIR, "cpt"); Index: branches/eam_branches/20090820/Ohana/src/dvomerge/src/dvomerge.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/dvomerge/src/dvomerge.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/dvomerge/src/dvomerge.c (revision 25766) @@ -24,5 +24,5 @@ // the first input define the photcode table & db layout sprintf (filename, "%s/Photcodes.dat", input1); - if (!LoadPhotcodes (filename, NULL)) { + if (!LoadPhotcodes (filename, NULL, FALSE)) { fprintf (stderr, "error loading photcode table %s\n", filename); exit (1); @@ -30,5 +30,5 @@ // save the photcodes in the output catdir sprintf (filename, "%s/Photcodes.dat", output); - if (!check_file_access (filename, TRUE, TRUE)) { + if (!check_file_access (filename, TRUE, TRUE, TRUE)) { fprintf (stderr, "error creating output catdir %s\n", output); exit (1); @@ -40,12 +40,12 @@ // load the sky table for the existing database - insky1 = SkyTableLoadOptimal (input1, NULL, NULL, SKY_DEPTH_HST, VERBOSE); + insky1 = SkyTableLoadOptimal (input1, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE); SkyTableSetFilenames (insky1, input1, "cpt"); - insky2 = SkyTableLoadOptimal (input2, NULL, NULL, SKY_DEPTH_HST, VERBOSE); + insky2 = SkyTableLoadOptimal (input2, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE); SkyTableSetFilenames (insky2, input2, "cpt"); // generate an output table populated at the desired depth - outsky = SkyTableLoadOptimal (output, NULL, GSCFILE, SKY_DEPTH, VERBOSE); + outsky = SkyTableLoadOptimal (output, NULL, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); SkyTableSetFilenames (outsky, output, "cpt"); @@ -117,5 +117,5 @@ // save the output sky table copy sprintf (filename, "%s/SkyTable.fits", output); - check_file_access (filename, TRUE, VERBOSE); + check_file_access (filename, TRUE, TRUE, VERBOSE); if (!SkyTableSave (outsky, filename)) { fprintf (stderr, "ERROR: failed to save sky table for %s\n", output); Index: branches/eam_branches/20090820/Ohana/src/dvosplit/src/ConfigInit.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/dvosplit/src/ConfigInit.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/dvosplit/src/ConfigInit.c (revision 25766) @@ -33,5 +33,5 @@ /* XXX this does not yet write out the master photcode table */ sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile)) { + if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) { fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); exit (1); Index: branches/eam_branches/20090820/Ohana/src/dvosplit/src/dvosplit.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/dvosplit/src/dvosplit.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/dvosplit/src/dvosplit.c (revision 25766) @@ -18,5 +18,5 @@ // load the sky table for the existing database - sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, SKY_DEPTH_HST, VERBOSE); + sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, TRUE, SKY_DEPTH_HST, VERBOSE); SkyTableSetFilenames (sky, CATDIR, "cpt"); @@ -79,5 +79,5 @@ // save sky table copy sprintf (filename, "%s/SkyTable.fits", CATDIR); - check_file_access (filename, TRUE, VERBOSE); + check_file_access (filename, TRUE, TRUE, VERBOSE); if (!SkyTableSave (sky, filename)) { fprintf (stderr, "ERROR: failed to save sky table for %s\n", CATDIR); Index: branches/eam_branches/20090820/Ohana/src/gastro/src/getptolemy.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/gastro/src/getptolemy.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/gastro/src/getptolemy.c (revision 25766) @@ -19,5 +19,5 @@ /* load regions from GSC table, restrict to patch */ - sky = SkyTableLoadOptimal (CATDIR, NULL, GSCFILE, SKY_DEPTH_HST, VERBOSE); + sky = SkyTableLoadOptimal (CATDIR, NULL, GSCFILE, FALSE, SKY_DEPTH_HST, VERBOSE); SkyTableSetFilenames (sky, CATDIR, "cpt"); skylist = SkyListByPatch (sky, -1, &patch); Index: branches/eam_branches/20090820/Ohana/src/gastro2/src/getptolemy.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/gastro2/src/getptolemy.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/gastro2/src/getptolemy.c (revision 25766) @@ -21,5 +21,5 @@ /* load regions from GSC table, restrict to patch */ - sky = SkyTableLoadOptimal (CATDIR, NULL, GSCFILE, SKY_DEPTH_HST, VERBOSE); + sky = SkyTableLoadOptimal (CATDIR, NULL, GSCFILE, FALSE, SKY_DEPTH_HST, VERBOSE); SkyTableSetFilenames (sky, CATDIR, "cpt"); skylist = SkyListByPatch (sky, -1, &patch); Index: branches/eam_branches/20090820/Ohana/src/getstar/include/dvoImagesAtCoords.h =================================================================== --- branches/eam_branches/20090820/Ohana/src/getstar/include/dvoImagesAtCoords.h (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/getstar/include/dvoImagesAtCoords.h (revision 25766) @@ -18,4 +18,13 @@ Coords *MOSAIC; // carries the mosaic into ReadImageHeader +typedef struct { + int id; + double ra; + double dec; + int Nmatches; + int *matches; + int arrayLength; +} Point; + int WITH_PHU; int SOLO_PHU; @@ -33,7 +42,5 @@ Image *ReadImageFiles (char *filename, int *Nimages); int ReadImageHeader (Header *header, Image *image); -int *MatchCoords(Image *, int, double, double, int *); +int MatchCoords(Image *, int, Point *, int); int GetFileMode (Header *header); -// int edge_check (double *x1, double *y1, double *x2, double *y2); -// double opening_angle (double x1, double y1, double x2, double y2, double x3, double y3); Index: branches/eam_branches/20090820/Ohana/src/getstar/src/ConfigInit.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/getstar/src/ConfigInit.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/getstar/src/ConfigInit.c (revision 25766) @@ -34,5 +34,5 @@ /* XXX this does not yet write out the master photcode table */ sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile)) { + if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) { fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); exit (1); Index: branches/eam_branches/20090820/Ohana/src/getstar/src/ConfigInit_coords.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/getstar/src/ConfigInit_coords.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/getstar/src/ConfigInit_coords.c (revision 25766) @@ -35,5 +35,5 @@ /* XXX this does not yet write out the master photcode table */ sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile)) { + if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) { fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); exit (1); Index: branches/eam_branches/20090820/Ohana/src/getstar/src/ConfigInit_extract.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/getstar/src/ConfigInit_extract.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/getstar/src/ConfigInit_extract.c (revision 25766) @@ -35,5 +35,5 @@ /* XXX this does not yet write out the master photcode table */ sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile)) { + if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) { fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); exit (1); Index: branches/eam_branches/20090820/Ohana/src/getstar/src/ConfigInit_overlaps.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/getstar/src/ConfigInit_overlaps.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/getstar/src/ConfigInit_overlaps.c (revision 25766) @@ -35,5 +35,5 @@ /* XXX this does not yet write out the master photcode table */ sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile)) { + if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, FALSE)) { fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); exit (1); Index: branches/eam_branches/20090820/Ohana/src/getstar/src/MatchCoords.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/getstar/src/MatchCoords.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/getstar/src/MatchCoords.c (revision 25766) @@ -5,8 +5,8 @@ /* given coordinate, find images in list that contain the point */ -int *MatchCoords (Image *dbImages, int NdbImages, double ra, double dec, int *Nmatch) { +int MatchCoords (Image *dbImages, int NdbImages, Point *points, int Npoints) { - int i, j, N, addtolist, status; - int NMATCH, nmatch, *match; + int i, j, N; + int totalMatches = 0; Coords tcoords; double r, d; @@ -15,10 +15,4 @@ double xmin, xmax, ymin, ymax; - *Nmatch = 0; - - /* match represents the subset of overlapping images */ - nmatch = 0; - NMATCH = 20; - ALLOCATE (match, int, NMATCH); /* setup links for mosaic WRP and DIS entries */ @@ -49,28 +43,32 @@ } - // transform input point to image coords - double x, y; - int status = RD_to_XY(&x, &y, ra, dec, &dbImages[i].coords); + for (j = 0; j < Npoints; j++) { + Point *pt = points + j; - if (!status) { - // avoid matching antipodal skycells - continue; - } + // transform input point to image coords + double x, y; + int status = RD_to_XY(&x, &y, pt->ra, pt->dec, &dbImages[i].coords); - if ((x >= xmin && x <= xmax) && (y >= ymin && y <= ymax)) { + if (!status) { + // avoid matching antipodal skycells + continue; + } - match[nmatch] = i; - nmatch ++; - if (nmatch == NMATCH) { - NMATCH += 20; - REALLOCATE (match, int, NMATCH); + if ((x >= xmin && x <= xmax) && (y >= ymin && y <= ymax)) { + totalMatches++; + + pt->matches[pt->Nmatches] = i; + pt->Nmatches ++; + + if (pt->Nmatches == pt->arrayLength) { + pt->arrayLength += 20; + REALLOCATE (pt->matches, int, pt->arrayLength); + } } } - - } - if (VERBOSE) fprintf (stderr, "found %d overlapping images\n", nmatch); + } + if (VERBOSE) fprintf (stderr, "found %d overlapping images\n", totalMatches); - *Nmatch = nmatch; - return (match); + return (totalMatches); } Index: branches/eam_branches/20090820/Ohana/src/getstar/src/dvoImagesAtCoords.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/getstar/src/dvoImagesAtCoords.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/getstar/src/dvoImagesAtCoords.c (revision 25766) @@ -1,12 +1,11 @@ # include "dvoImagesAtCoords.h" -typedef struct { - int id; - double ra; - double dec; -} Point; +// We'd like to do this but since pstamp gets built later this won't work +// so just redefine the macro +// # include "pstamp.h" +#define PSTAMP_NO_OVERLAP 28 static int readPoints(char *filename, Point **pointsOut); -static int ListImagesAtCoords (Image *dbImages, int id, double ra, double dec, int *matches, int Nmatches); +static int ListImagesAtCoords (Image *dbImages, Point *points, int Npoints); int main (int argc, char **argv) { @@ -25,10 +24,10 @@ Npoints = readPoints(argv[1], &points); if (!Npoints) { - exit(0); + exit(1); } if (astromFile) { dbImages = ReadImageFiles(astromFile, &NdbImages); - } else { + } else { /*** update the image table ***/ /* setup image table format and lock */ @@ -53,11 +52,10 @@ } - Point *pt; - for (i = 0, pt = points; i < Npoints; i++, pt++) { - matches = MatchCoords (dbImages, NdbImages, pt->ra, pt->dec, &Nmatches); - ListImagesAtCoords(dbImages, pt->id, pt->ra, pt->dec, matches, Nmatches); + if (MatchCoords (dbImages, NdbImages, points, Npoints)) { + ListImagesAtCoords(dbImages, points, Npoints); + exit(0); + } else { + exit(PSTAMP_NO_OVERLAP); } - - exit (0); } @@ -83,6 +81,9 @@ while ((Nread = fscanf(f, "%d %lf %lf\n", &pts[Npoints].id, &pts[Npoints].ra, &pts[Npoints].dec)) == 3) { - Npoints++; - if (Npoints >= LEN) { + pts[Npoints].Nmatches = 0; + ALLOCATE(pts[Npoints].matches, int, 20); + pts[Npoints].arrayLength = 20; + + if (++Npoints >= LEN) { LEN += 20; REALLOCATE(pts, Point, LEN); @@ -99,24 +100,33 @@ } -static int ListImagesAtCoords (Image *dbImages, int id, double ra, double dec, int *matches, int Nmatches) +static int ListImagesAtCoords (Image *dbImages, Point *points, int Npoints) { + int j; int i; - for (i = 0; i < Nmatches; i++) { - int N = matches[i]; + for (j = 0; j < Npoints; j++) { + Point *pt = points + j; + for (i = 0; i < pt->Nmatches; i++) { + int N = pt->matches[i]; - char *name; - // output of lookup is filename[class_id.hdr] for astrometry files - char *left_bracket = rindex(dbImages[N].name, '['); - if (left_bracket) { - name = strdup(left_bracket + 1); - // zap the .hdr] - char *dot = index(name, '.'); - if (dot) { - *dot = 0; + char *name; + char *copy = NULL; + // output of lookup is filename[class_id.hdr] for astrometry files + char *left_bracket = rindex(dbImages[N].name, '['); + if (left_bracket) { + copy = strdup(left_bracket + 1); + name = copy; + // zap the .hdr] + char *dot = index(name, '.'); + if (dot) { + *dot = 0; + } + } else { + name = dbImages[N].name; } - } else { - name = dbImages[N].name; + fprintf (stdout, "%d %lf %lf %s\n", pt->id, pt->ra, pt->dec, name); + if (copy) { + free(copy); + } } - fprintf (stdout, "%d %lf %lf %s\n", id, ra, dec, name); } Index: branches/eam_branches/20090820/Ohana/src/getstar/src/getstar.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/getstar/src/getstar.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/getstar/src/getstar.c (revision 25766) @@ -13,5 +13,5 @@ set_db (&db); - sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, SKY_DEPTH, VERBOSE); + sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); if (!sky) exit (1); Index: branches/eam_branches/20090820/Ohana/src/imregister/base/ConfigInit.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/imregister/base/ConfigInit.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/imregister/base/ConfigInit.c (revision 25766) @@ -105,5 +105,5 @@ /* XXX this does not yet write out the master photcode table */ sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", catdir); - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile)) { + if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) { fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); exit (1); Index: branches/eam_branches/20090820/Ohana/src/imregister/imphot/ConfigInit.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/imregister/imphot/ConfigInit.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/imregister/imphot/ConfigInit.c (revision 25766) @@ -101,5 +101,5 @@ /* XXX this does not yet write out the master photcode table */ sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile)) { + if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) { fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); exit (1); Index: branches/eam_branches/20090820/Ohana/src/kapa2/Makefile =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/Makefile (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/Makefile (revision 25766) @@ -48,7 +48,8 @@ $(SRC)/Layout.$(ARCH).o $(SRC)/Sections.$(ARCH).o \ $(SRC)/Graphs.$(ARCH).o $(SRC)/SetGraphSize.$(ARCH).o \ -$(SRC)/Resize.$(ARCH).o $(SRC)/ErasePlots.$(ARCH).o \ -$(SRC)/EraseImage.$(ARCH).o $(SRC)/SetToolbox.$(ARCH).o \ +$(SRC)/Resize.$(ARCH).o $(SRC)/Relocate.$(ARCH).o \ +$(SRC)/ErasePlots.$(ARCH).o $(SRC)/EraseImage.$(ARCH).o \ $(SRC)/EraseCurrentPlot.$(ARCH).o $(SRC)/EraseSections.$(ARCH).o \ +$(SRC)/SetToolbox.$(ARCH).o \ $(SRC)/SetSection.$(ARCH).o $(SRC)/DefineSection.$(ARCH).o \ $(SRC)/SetLimits.$(ARCH).o $(SRC)/SetFont.$(ARCH).o \ @@ -79,7 +80,4 @@ $(SRC)/ColorHistogram.$(ARCH).o $(SRC)/CreateWide.$(ARCH).o -#$(SRC)/CreateZoom8.$(ARCH).o $(SRC)/CreateZoom16.$(ARCH).o \ -#$(SRC)/CreateZoom24.$(ARCH).o $(SRC)/CreateZoom32.$(ARCH).o \ - OBJ = $(KAPA) Index: branches/eam_branches/20090820/Ohana/src/kapa2/include/constants.h =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/include/constants.h (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/include/constants.h (revision 25766) @@ -11,5 +11,5 @@ # define NCHANNELS 3 -# define NPIXELS_DYNAMIC 128 +# define NPIXELS_DYNAMIC 4600 // XXX for the moment, this is set to match the values in SetColorScale3D_CC Index: branches/eam_branches/20090820/Ohana/src/kapa2/include/prototypes.h =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/include/prototypes.h (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/include/prototypes.h (revision 25766) @@ -55,4 +55,5 @@ int LoadTextlines PROTO((int sock)); int Resize PROTO((int sock)); +int Relocate PROTO((int sock)); int GetLimits PROTO((int sock)); int SetLimits PROTO((int sock)); @@ -178,6 +179,6 @@ void Image_to_Picture PROTO((double *x1, double *y1, double x2, double y2, Picture *picture)); void Image_to_Screen PROTO((double *x1, double *y1, double x2, double y2, Picture *picture)); -void Picture_Lower PROTO((int *i_start, int *j_start, Matrix *matrix, Picture *picture)); -void Picture_Upper PROTO((int *i_end, int *j_end, Matrix *matrix, Picture *picture)); +void Picture_Lower PROTO((int *i_start, int *j_start, int *I_start, int *J_start, Matrix *matrix, Picture *picture)); +void Picture_Upper PROTO((int *i_end, int *j_end, int i_start, int j_start, Matrix *matrix, Picture *picture)); void DragColorbar PROTO((Graphic *graphic, KapaImageWidget *image, XButtonEvent *mouse_event)); Index: branches/eam_branches/20090820/Ohana/src/kapa2/include/structures.h =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/include/structures.h (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/include/structures.h (revision 25766) @@ -21,5 +21,5 @@ Window window; Visual *visual; - int visualclass; // is visual dynamic? (XXX change name?) + int dynamicColors; // is visual dynamic? int Nbits; // pixel depth in bits (8, 16, 24, 32) GC gc; @@ -94,5 +94,5 @@ int DX, DY; // size of displayed picture (must be updated with new images...) int expand; // zoomscale - double X, Y; // center of image in picture + double Xc, Yc; // center of image in picture char flipx, flipy; // parity (0 = +; 1 = -) XImage *pix; Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/ButtonFunctions.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/ButtonFunctions.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/ButtonFunctions.c (revision 25766) @@ -44,6 +44,6 @@ int Recenter (Graphic *graphic, KapaImageWidget *image) { - image[0].picture.X = 0.5*image[0].image[0].matrix.Naxis[0]; - image[0].picture.Y = 0.5*image[0].image[0].matrix.Naxis[1]; + image[0].picture.Xc = 0.5*image[0].image[0].matrix.Naxis[0]; + image[0].picture.Yc = 0.5*image[0].image[0].matrix.Naxis[1]; Remap (graphic, image); @@ -56,4 +56,5 @@ image[0].picture.expand = 1; + image[0].zoom.expand = 5; Remap (graphic, image); Refresh (); @@ -64,7 +65,8 @@ int RecenterRescale (Graphic *graphic, KapaImageWidget *image) { - image[0].picture.X = 0.5*image[0].image[0].matrix.Naxis[0]; - image[0].picture.Y = 0.5*image[0].image[0].matrix.Naxis[1]; + image[0].picture.Xc = 0.5*image[0].image[0].matrix.Naxis[0]; + image[0].picture.Yc = 0.5*image[0].image[0].matrix.Naxis[1]; image[0].picture.expand = 1; + image[0].zoom.expand = 5; Remap (graphic, image); Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/Center.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/Center.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/Center.c (revision 25766) @@ -16,8 +16,10 @@ if (image == NULL) return (TRUE); - image[0].picture.X = X; - image[0].picture.Y = Y; + // enforce integer center here? + image[0].picture.Xc = X; + image[0].picture.Yc = Y; if ((zoom != 0) && (zoom != -1)) { image[0].picture.expand = zoom; + image[0].zoom.expand = MIN(image[0].zoom.dx / 5.5, MAX(5, 2*zoom)); } Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/CheckPipe.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/CheckPipe.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/CheckPipe.c (revision 25766) @@ -131,4 +131,10 @@ } + if (!strcmp (word, "MOVE")) { + status = Relocate (sock); + KiiSendCommand (sock, 4, "DONE"); + FINISHED (status); + } + if (!strcmp (word, "GLIM")) { GetLimits (sock); Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/CheckVisual.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/CheckVisual.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/CheckVisual.c (revision 25766) @@ -10,5 +10,5 @@ int i, Nfound, N; - int isColor, isDefault, isDynamic; + int isColor, isDefault; XVisualInfo *visual_list, visual_temp; unsigned long planes[3]; @@ -34,29 +34,8 @@ } -// skip the default and the PseudoColor visuals, go for TrueColor first -# if (0) - /* check default visual first */ - for (i = 0; (i < Nfound) && (graphic[0].visual != visual_list[i].visual); i++); - if (i == Nfound) { - fprintf (stderr, "default visual not found??\n"); - exit (0); - } -# endif - // set these based on selected visual - isColor = isDefault = isDynamic = FALSE; - -# if (0) - // attempt to select the most desirable type of visual: Default as PseudoColor (XXX is it still true?) - if (DEBUG) fprintf (stderr, "default visual class is %d\n", visual_list[i].class); - if (visual_list[i].class == PseudoColor) { - isColor = isDefault = isDynamic = TRUE; - graphic[0].visual = visual_list[i].visual; - graphic[0].visualclass = TRUE; - goto test_pixels; - } -# endif - - // attempt to select the most desirable type of visual: TrueColor (XXX is it still true?) + isColor = isDefault = FALSE; + + // attempt to select the most desirable type of visual: TrueColor for (i = 0; (i < Nfound) && (visual_list[i].class != TrueColor); i++); if (i != Nfound) { @@ -65,5 +44,18 @@ isDefault = (graphic[0].visual == visual_list[i].visual); graphic[0].visual = visual_list[i].visual; - graphic[0].visualclass = FALSE; + graphic[0].dynamicColors = FALSE; + if (DEBUG) fprintf (stderr, "got TrueColor visual\n"); + goto test_pixels; + } + + // attempt to select the most desirable type of visual: DirectColor + for (i = 0; (i < Nfound) && (visual_list[i].class != DirectColor); i++); + if (i != Nfound) { + isColor = TRUE; + if (DEBUG) fprintf (stderr, "visual class is %d\n", visual_list[i].class); + isDefault = (graphic[0].visual == visual_list[i].visual); + graphic[0].visual = visual_list[i].visual; + graphic[0].dynamicColors = TRUE; + if (DEBUG) fprintf (stderr, "got DirectColor visual\n"); goto test_pixels; } @@ -72,9 +64,10 @@ for (i = 0; (i < Nfound) && (visual_list[i].class != PseudoColor); i++); if (i != Nfound) { - isColor = isDynamic = TRUE; + isColor = TRUE; if (DEBUG) fprintf (stderr, "selected visual class is %d\n", visual_list[i].class); isDefault = (graphic[0].visual == visual_list[i].visual); graphic[0].visual = visual_list[i].visual; - graphic[0].visualclass = TRUE; + graphic[0].dynamicColors = TRUE; + if (DEBUG) fprintf (stderr, "got PseudoColor visual\n"); goto test_pixels; } @@ -83,9 +76,9 @@ for (i = 0; (i < Nfound) && (visual_list[i].class != GrayScale); i++); if (i != Nfound) { - isDynamic = TRUE; if (DEBUG) fprintf (stderr, "selected visual class is %d\n", visual_list[i].class); isDefault = (graphic[0].visual == visual_list[i].visual); graphic[0].visual = visual_list[i].visual; - graphic[0].visualclass = TRUE; + graphic[0].dynamicColors = TRUE; + if (DEBUG) fprintf (stderr, "got GrayScale visual\n"); goto test_pixels; } @@ -97,5 +90,6 @@ isDefault = (graphic[0].visual == visual_list[i].visual); graphic[0].visual = visual_list[i].visual; - graphic[0].visualclass = FALSE; + graphic[0].dynamicColors = FALSE; + if (DEBUG) fprintf (stderr, "got StaticColor visual\n"); goto test_pixels; } @@ -107,5 +101,6 @@ isDefault = (graphic[0].visual == visual_list[i].visual); graphic[0].visual = visual_list[i].visual; - graphic[0].visualclass = FALSE; + graphic[0].dynamicColors = FALSE; + if (DEBUG) fprintf (stderr, "got StaticGray visual\n"); goto test_pixels; } @@ -115,4 +110,5 @@ /* need to make a colormap if 1) the selected visual is not the default + XXX not sure if I need to / am allowed to alloc a colormap if I've grabbed the default. 2) there are not enough colors available */ @@ -125,11 +121,15 @@ } - // allocate a private colormap, if needed + // dynamic visual classes can accept AllocAll and can use XAllocColorCells while AllocNone must use XAllocColor + // int allocMode = graphic[0].dynamicColors ? AllocAll : AllocNone; + int allocMode = AllocNone; + + // allocate a private colormap, if desired or needed if (!isDefault) { if (DEBUG) fprintf (stderr, "allocated private colormap\n"); - graphic[0].colormap = XCreateColormap (graphic[0].display, RootWindow (graphic[0].display, graphic[0].screen), graphic[0].visual, AllocNone); - } - - if (isDynamic) { + graphic[0].colormap = XCreateColormap (graphic[0].display, RootWindow (graphic[0].display, graphic[0].screen), graphic[0].visual, allocMode); + } + + if (graphic[0].dynamicColors) { Npixels = NPIXELS_DYNAMIC; if ((N = get_argument (*argc, argv, "-ncolors"))) { @@ -144,6 +144,9 @@ for (graphic[0].Npixels = Npixels; graphic[0].Npixels >= 16; graphic[0].Npixels -= 4) { if (DEBUG) fprintf (stderr, "trying %d colors\n", (int) graphic[0].Npixels); - if (XAllocColorCells (graphic[0].display, graphic[0].colormap, FALSE, planes, 1, graphic[0].pixels, graphic[0].Npixels)) { + if (XAllocColorCells (graphic[0].display, graphic[0].colormap, FALSE, planes, 0, graphic[0].pixels, graphic[0].Npixels)) { break; + } + for (i = 0; i < graphic[0].Npixels; i++) { + graphic[0].cmap[i].pixel = graphic[0].pixels[i]; } } @@ -152,9 +155,10 @@ if (graphic[0].Npixels < 16) { if (!isDefault) { + // We've already tried to allocate a colormap above... fprintf (stderr, "can't allocate enough cells in private colormap\n"); exit (0); } if (DEBUG) fprintf (stderr, "can't allocate enough cells, using private colormap\n"); - graphic[0].colormap = XCreateColormap (graphic[0].display, RootWindow (graphic[0].display, graphic[0].screen), graphic[0].visual, AllocNone); + graphic[0].colormap = XCreateColormap (graphic[0].display, RootWindow (graphic[0].display, graphic[0].screen), graphic[0].visual, allocMode); for (graphic[0].Npixels = NPIXELS_DYNAMIC; graphic[0].Npixels >= 16; graphic[0].Npixels -= 4) { @@ -163,6 +167,8 @@ break; } - } - + for (i = 0; i < graphic[0].Npixels; i++) { + graphic[0].cmap[i].pixel = graphic[0].pixels[i]; + } + } if ((N = get_argument (*argc, argv, "-colorcount"))) { fprintf (stderr, "kapa can grab %d colors\n", graphic[0].Npixels); Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/CheckVisual.test.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/CheckVisual.test.c (revision 25766) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/CheckVisual.test.c (revision 25766) @@ -0,0 +1,174 @@ +# include "Ximage.h" + +/* DirectColor doesn't seem to work, even though it is available: + I cannot use XAllocColorCells to get pixels under DirectColor +*/ + +/* static int try_visual[] = {5, 3, 1, 4, 2, 0}; */ + +int TryVisualType (int *isColor, int *isDefault, Graphic *graphic, int Nvisual, XVisualInfo *visual_list, int VisualType, char *name) { + + int i; + + for (i = 0; i < Nvisual; i++) { + if (visual_list[i].class != VisualType) continue; + + *isColor = FALSE; + if (VisualType == DirectColor) *isColor = TRUE; + if (VisualType == PseudoColor) *isColor = TRUE; + if (VisualType == TrueColor) *isColor = TRUE; + + if (DEBUG) fprintf (stderr, "visual class is %d\n", visual_list[i].class); + *isDefault = (graphic[0].visual == visual_list[i].visual); + + graphic[0].dynamicColors = FALSE; + if (VisualType == DirectColor) graphic[0].dynamicColors = TRUE; + if (VisualType == PseudoColor) graphic[0].dynamicColors = TRUE; + if (VisualType == GrayScale) graphic[0].dynamicColors = TRUE; + + if (DEBUG) fprintf (stderr, "got %s visual\n", name); + return (TRUE); + } + return (FALSE); +} + +void CheckVisual (Graphic *graphic, int *argc, char **argv) { + + int i, Nvisual, N; + int isColor, isDefault; + XVisualInfo *visual_list, visual_temp; + unsigned long planes[3]; + XPixmapFormatValues *pixmaps; + int Npixmaps, Npixels; + + if (DEBUG) { + fprintf (stderr, "DirectColor: %d\n", DirectColor); + fprintf (stderr, "PseudoColor: %d\n", PseudoColor); + fprintf (stderr, "TrueColor: %d\n", TrueColor); + fprintf (stderr, "GrayScale: %d\n", GrayScale); + fprintf (stderr, "StaticColor: %d\n", StaticColor); + fprintf (stderr, "StaticGray: %d\n", StaticGray); + } + + visual_temp.screen = graphic[0].screen; + + /* find available visuals */ + visual_list = XGetVisualInfo (graphic[0].display, VisualScreenMask, &visual_temp, &Nvisual); + if (Nvisual == 0) { + fprintf (stderr, "error finding useful visual\n"); + exit (2); + } + + // set these based on selected visual + isColor = isDefault = FALSE; + + if (TryVisualType (&isColor, &isDefault, graphic, Nvisual, visual_list, TrueColor, "TrueColor")) goto test_pixels; + if (TryVisualType (&isColor, &isDefault, graphic, Nvisual, visual_list, DirectColor, "DirectColor")) goto test_pixels; + if (TryVisualType (&isColor, &isDefault, graphic, Nvisual, visual_list, PseudoColor, "PseudoColor")) goto test_pixels; + if (TryVisualType (&isColor, &isDefault, graphic, Nvisual, visual_list, GrayScale, "GrayScale")) goto test_pixels; + if (TryVisualType (&isColor, &isDefault, graphic, Nvisual, visual_list, StaticColor, "StaticColor")) goto test_pixels; + if (TryVisualType (&isColor, &isDefault, graphic, Nvisual, visual_list, StaticGray, "StaticGray")) goto test_pixels; + + fprintf (stderr, "error finding valid visual\n"); + exit (2); + + test_pixels: + + /* need to make a colormap if + 1) the selected visual is not the default + XXX not sure if I need to / am allowed to alloc a colormap if I've grabbed the default. + 2) there are not enough colors available */ + + /* NEED TO ADD A COUPLE LINES DEFINING THE VARIOUS HARD COLORS (BLACK, WHITE, AND THE OVERLAYS) */ + + // allow user to force a private colormap + if ((N = get_argument (*argc, argv, "-private"))) { + remove_argument(N, argc, argv); + isDefault = FALSE; + } + + // dynamic visual classes can accept AllocAll and can use XAllocColorCells while AllocNone must use XAllocColor + // int allocMode = graphic[0].dynamicColors ? AllocAll : AllocNone; + int allocMode = AllocNone; + + // allocate a private colormap, if desired or needed + if (!isDefault) { + if (DEBUG) fprintf (stderr, "allocated private colormap\n"); + graphic[0].colormap = XCreateColormap (graphic[0].display, RootWindow (graphic[0].display, graphic[0].screen), graphic[0].visual, allocMode); + } + + if (graphic[0].dynamicColors) { + Npixels = NPIXELS_DYNAMIC; + if ((N = get_argument (*argc, argv, "-ncolors"))) { + remove_argument(N, argc, argv); + Npixels = atoi (argv[N]); + remove_argument(N, argc, argv); + } + + /* allocate color cells */ + ALLOCATE (graphic[0].pixels, unsigned long, Npixels); + ALLOCATE (graphic[0].cmap, XColor, Npixels); + for (graphic[0].Npixels = Npixels; graphic[0].Npixels >= 16; graphic[0].Npixels -= 4) { + if (DEBUG) fprintf (stderr, "trying %d colors\n", (int) graphic[0].Npixels); + if (XAllocColorCells (graphic[0].display, graphic[0].colormap, FALSE, planes, 0, graphic[0].pixels, graphic[0].Npixels)) { + break; + } + for (i = 0; i < graphic[0].Npixels; i++) { + graphic[0].cmap[i].pixel = graphic[0].pixels[i]; + } + } + + /* insufficient cells, can we make a private colormap? */ + if (graphic[0].Npixels < 16) { + if (!isDefault) { + // We've already tried to allocate a colormap above... + fprintf (stderr, "can't allocate enough cells in private colormap\n"); + exit (0); + } + if (DEBUG) fprintf (stderr, "can't allocate enough cells, using private colormap\n"); + graphic[0].colormap = XCreateColormap (graphic[0].display, RootWindow (graphic[0].display, graphic[0].screen), graphic[0].visual, allocMode); + + for (graphic[0].Npixels = NPIXELS_DYNAMIC; graphic[0].Npixels >= 16; graphic[0].Npixels -= 4) { + if (DEBUG) fprintf (stderr, "trying %d colors\n", (int) graphic[0].Npixels); + if (XAllocColorCells (graphic[0].display, graphic[0].colormap, FALSE, planes, 1, graphic[0].pixels, graphic[0].Npixels)) { + break; + } + for (i = 0; i < graphic[0].Npixels; i++) { + graphic[0].cmap[i].pixel = graphic[0].pixels[i]; + } + } + if ((N = get_argument (*argc, argv, "-colorcount"))) { + fprintf (stderr, "kapa can grab %d colors\n", graphic[0].Npixels); + exit (0); + } + if (graphic[0].Npixels < 16) { + fprintf (stderr, "can't even allocate enough cells in private colormap\n"); + exit (0); + } + } + } else { + Npixels = NPIXELS_STATIC; + if ((N = get_argument (*argc, argv, "-ncolors"))) { + remove_argument(N, argc, argv); + Npixels = atoi (argv[N]); + remove_argument(N, argc, argv); + } + + // XXX allocate the unsigned long *pixels here and above + ALLOCATE (graphic[0].pixels, unsigned long, Npixels); + ALLOCATE (graphic[0].cmap, XColor, Npixels); + graphic[0].Npixels = Npixels; + } + + if ((N = get_argument (*argc, argv, "-colorcount"))) { + fprintf (stderr, "kii can grab %d colors\n", graphic[0].Npixels); + exit (0); + } + + pixmaps = XListPixmapFormats (graphic[0].display, &Npixmaps); + for (i = 0; i < Npixmaps; i++) { + if (pixmaps[i].depth == graphic[0].depth) { + graphic[0].Nbits = pixmaps[i].bits_per_pixel; + } + } +} Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/CreateZoom16.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/CreateZoom16.c (revision 25206) +++ (revision ) @@ -1,144 +1,0 @@ -# include "Ximage.h" - -void CreateZoom16 (KapaImageWidget *image, Graphic *graphic, double x, double y) { - -# if (0) - int i, j, ii, jj; - int i_start, i_end, j_start, j_end; - int dropback; /* this is a bit of a kludge... */ - int dx, dy, DX, DY, pixelN; - double expand, zoomscale, Rx, Ry; - int expand_in, expand_out; - unsigned char *out_pix, *out_pix2, *data; - float *imdata, *in_pix, *in_pix2; - unsigned char pixel1[256], pixel2[256]; - unsigned char pixvalue1, pixvalue2; - unsigned long back; - unsigned char back1, back2; - - dx = image[0].zoom.dx; - dy = image[0].zoom.dy; - - if (image[0].image[0].matrix.size == 0) { /* create a test pattern */ - REALLOCATE (image[0].zoom.data, char, 2*dy*dx); - bzero (image[0].zoom.data, image[0].zoom.dx*image[0].zoom.dy); - image[0].zoom.pix = XCreateImage (graphic[0].display, graphic[0].visual, graphic[0].depth, ZPixmap, 0, - image[0].zoom.data, image[0].zoom.dx, image[0].zoom.dy, 16, 0); - return; - } - - for (i = 0; i < 256; i++) { /* set up pixel array */ - pixel1[i] = 0x0000ff & graphic[0].cmap[i].pixel; - pixel2[i] = 0x0000ff & (graphic[0].cmap[i].pixel >> 8); - } - back = graphic[0].back; - back1 = 0x0000ff & back; - back2 = 0x0000ff & (back >> 8); - // define the color transform parameters - MaxValue = graphic[0].Npixels - 1; - if (image[0].image[0].range != 0.0) { - slope = graphic[0].Npixels / image[0].image[0].range; - start = graphic[0].Npixels * image[0].image[0].zero / image[0].image[0].range; - } else { - slope = 1.0; - start = image[0].image[0].zero; - } - - zoomscale = MAX (5, image[0].expand + 5); - expand = 1 / zoomscale; - expand_out = (int) zoomscale; - expand_in = 1; - - DX = image[0].image[0].matrix.Naxis[0]; - DY = image[0].image[0].matrix.Naxis[1]; - Rx = x - expand*(int)(0.5*(dx + 1)) + 1; - Ry = y - expand*(int)(0.5*(dy + 1)) + 1; - - i_start = MIN (MAX (-Rx / expand, (1 - FRAC(Rx)) / expand), dx - expand_out + 1); - j_start = MIN (MAX (-Ry / expand, (1 - FRAC(Ry)) / expand), dy - expand_out + 1); - i_end = MAX (MIN ((DX-Rx) / expand, dx - expand_out + 1), 0); - j_end = MAX (MIN ((DY-Ry) / expand, dy - expand_out + 1), 0); - dropback = expand_out - (i_end - i_start) % expand_out; - if ((i_end - i_start) % expand_out == 0) dropback = 0; - - data = out_pix = (unsigned char *) image[0].zoom.data; - imdata = (float *) image[0].image[0].matrix.buffer; - in_pix = &imdata[DX*(int)MAX(Ry,0) + (int)MAX(Rx,0)]; - - /********** below we do the mapping from buffer pixels (in) to picture pixels (out) **********/ - - if ((i_end < i_start) || (j_end < j_start)) { - for (j = 0; j < dx*dy; j++, out_pix++) - *out_pix = back; - return; - } - - /**** fill in bottom area ****/ - for (j = 0; j < j_start; j++) { - for (i = 0; i < dx; i++, out_pix+=2) { - out_pix[0] = back1; - out_pix[1] = back2; - } - } - - for (j = j_start; j < j_end; j+= expand_out, in_pix += expand_in*DX) { - out_pix = &data[2*j*dx]; - - /**** fill in area to the left of the picture ****/ - for (jj = 0; jj < expand_out; jj++) { - out_pix2 = out_pix + 2*jj*dx; - for (i = 0; i < i_start; i++, out_pix2+=2) { - out_pix2[0] = back1; - out_pix2[1] = back2; - } - } - out_pix += 2*i_start; - - /*** fill in the picture region ***/ - in_pix2 = in_pix; - if (expand_out == 1) { - for (i = i_start; i < i_end; i++, in_pix2+= expand_in, out_pix+=2) { - pixelN = PixelLookup(*in_pix2); - out_pix[0] = pixel1[pixelN]; - out_pix[1] = pixel2[pixelN]; - } - } else { - for (i = i_start; i < i_end; i+= expand_out, in_pix2++, out_pix+= 2*expand_out) { - pixelN = PixelLookup(*in_pix2); - pixvalue1 = pixel1[pixelN]; - pixvalue2 = pixel2[pixelN]; - out_pix2 = out_pix; - for (jj = 0; jj < expand_out; jj++, out_pix2+=2*(dx-expand_out)) { - for (ii = 0; ii < expand_out; ii++, out_pix2+=2) { - out_pix2[0] = pixvalue1; - out_pix2[1] = pixvalue2; - } - } - } - } - out_pix -= 2*dropback; - - /**** fill in area to the right of the picture ****/ - for (jj = 0; jj < expand_out; jj++) { - out_pix2 = out_pix + 2*jj*dx; - for (i = i_end; i < dx; i++, out_pix2+=2) { - out_pix2[0] = back1; - out_pix2[1] = back2; - } - } - } - - out_pix = &data[2*j_end*dx]; - /**** fill in top area ****/ - for (j = 0; j < (dy - j_end); j++) { - for (i = 0; i < dx; i++, out_pix+=2) { - out_pix[0] = back1; - out_pix[1] = back2; - } - } - - image[0].zoom.pix = XCreateImage (graphic[0].display, graphic[0].visual, graphic[0].depth, ZPixmap, 0, - image[0].zoom.data, image[0].zoom.dx, image[0].zoom.dy, 16, 0); - -# endif -} Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/CreateZoom24.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/CreateZoom24.c (revision 25206) +++ (revision ) @@ -1,178 +1,0 @@ -# include "Ximage.h" - -void CreateZoom24 (KapaImageWidget *image, Graphic *graphic, double x, double y) { - -# if (0) - int i, j, ii, jj, extra; - int i_start, i_end, j_start, j_end; - int dropback; /* this is a bit of a kludge... */ - int dx, dy, DX, DY, pixelN; - double expand, zoomscale, Rx, Ry; - int expand_in, expand_out; - unsigned char *out_pix, *out_pix2, *data; - float *imdata, *in_pix, *in_pix2; - unsigned char pixel1[256], pixel2[256], pixel3[256]; - unsigned char pixvalue1, pixvalue2, pixvalue3; - unsigned long back; - unsigned char back1, back2, back3; - - dx = image[0].zoom.dx; - dy = image[0].zoom.dy; - extra = 4 - (dx * 3) % 4; - - if (image[0].image[0].matrix.size == 0) { /* create a test pattern */ - REALLOCATE (image[0].zoom.data, char, dy*(3*dx+extra)); - bzero (image[0].zoom.data, image[0].zoom.dx*image[0].zoom.dy); - image[0].zoom.pix = XCreateImage (graphic[0].display, graphic[0].visual, graphic[0].depth, ZPixmap, 0, - image[0].zoom.data, image[0].zoom.dx, image[0].zoom.dy, 32, 0); - return; - } - - for (i = 0; i < 256; i++) { /* set up pixel array */ - pixel1[i] = 0x0000ff & graphic[0].cmap[i].pixel; - pixel2[i] = 0x0000ff & (graphic[0].cmap[i].pixel >> 8); - pixel3[i] = 0x0000ff & (graphic[0].cmap[i].pixel >> 16); - } - back = graphic[0].back; - back1 = 0x0000ff & back; - back2 = 0x0000ff & (back >> 8); - back3 = 0x0000ff & (back >> 16); - - // define the color transform parameters - MaxValue = graphic[0].Npixels - 1; - if (image[0].image[0].range != 0.0) { - slope = graphic[0].Npixels / image[0].image[0].range; - start = graphic[0].Npixels * image[0].image[0].zero / image[0].image[0].range; - } else { - slope = 1.0; - start = image[0].image[0].zero; - } - - zoomscale = MAX (5, image[0].expand + 5); - expand = 1 / zoomscale; - expand_out = (int) zoomscale; - expand_in = 1; - - DX = image[0].image[0].matrix.Naxis[0]; - DY = image[0].image[0].matrix.Naxis[1]; - Rx = x - expand*(int)(0.5*(dx + 1)) + 1; - Ry = y - expand*(int)(0.5*(dy + 1)) + 1; - - i_start = MIN (MAX (-Rx / expand, (1 - FRAC(Rx)) / expand), dx - expand_out + 1); - j_start = MIN (MAX (-Ry / expand, (1 - FRAC(Ry)) / expand), dy - expand_out + 1); - i_end = MAX (MIN ((DX-Rx) / expand, dx - expand_out + 1), 0); - j_end = MAX (MIN ((DY-Ry) / expand, dy - expand_out + 1), 0); - dropback = expand_out - (i_end - i_start) % expand_out; - if ((i_end - i_start) % expand_out == 0) dropback = 0; - - data = out_pix = (unsigned char *) image[0].zoom.data; - imdata = (float *) image[0].image[0].matrix.buffer; - in_pix = &imdata[DX*(int)MAX(Ry,0) + (int)MAX(Rx,0)]; - - /********** below we do the mapping from buffer pixels (in) to picture pixels (out) **********/ - - if ((i_end < i_start) || (j_end < j_start)) { - for (j = 0; j < dx*dy; j++, out_pix++) - *out_pix = back; - return; - } - - /**** fill in bottom area ****/ - for (j = 0; j < j_start; j++) { - for (i = 0; i < dx; i++, out_pix+=3) { - out_pix[0] = back1; - out_pix[1] = back2; - out_pix[2] = back3; - } - out_pix += extra; - } - - for (j = j_start; j < j_end; j+= expand_out, in_pix += expand_in*DX) { - out_pix = &data[j*(3*dx+extra)]; - - /**** fill in area to the left of the picture ****/ - for (jj = 0; jj < expand_out; jj++) { - out_pix2 = out_pix + jj*(3*dx + extra); - for (i = 0; i < i_start; i++, out_pix2+=3) { - out_pix2[0] = back1; - out_pix2[1] = back2; - out_pix2[2] = back3; - } - } - out_pix += 3*i_start; - - /*** fill in the picture region ***/ - in_pix2 = in_pix; - if (expand_out == 1) { - for (i = i_start; i < i_end; i++, in_pix2+= expand_in, out_pix+=3) { - pixelN = PixelLookup(*in_pix2); - out_pix[0] = pixel1[pixelN]; - out_pix[1] = pixel2[pixelN]; - out_pix[2] = pixel3[pixelN]; - } - } else { - for (i = i_start; i < i_end; i+= expand_out, in_pix2++, out_pix+= 3*expand_out) { - pixelN = PixelLookup(*in_pix2); - pixvalue1 = pixel1[pixelN]; - pixvalue2 = pixel2[pixelN]; - pixvalue3 = pixel3[pixelN]; - out_pix2 = out_pix; - for (jj = 0; jj < expand_out; jj++, out_pix2+=3*(dx-expand_out)+extra) { - for (ii = 0; ii < expand_out; ii++, out_pix2+=3) { - out_pix2[0] = pixvalue1; - out_pix2[1] = pixvalue2; - out_pix2[2] = pixvalue3; - } - } - } - } - out_pix -= 3*dropback; - - /**** fill in area to the right of the picture ****/ - for (jj = 0; jj < expand_out; jj++) { - out_pix2 = out_pix + jj*(3*dx+extra); - for (i = i_end; i < dx; i++, out_pix2+=3) { - out_pix2[0] = back1; - out_pix2[1] = back2; - out_pix2[2] = back3; - } - } - } - - /* - if ((j_end - j_start) % expand_out > 0) - out_pix -= (expand_out - (j_end - j_start) % expand_out); - */ - - out_pix = &data[j_end*(3*dx+extra)]; - /**** fill in top area ****/ - for (j = 0; j < (dy - j_end); j++) { - for (i = 0; i < dx; i++, out_pix+=3) { - out_pix[0] = back1; - out_pix[1] = back2; - out_pix[2] = back3; - } - out_pix+=extra; - } - - /* - out_pix = (unsigned char *) image[0].zoom.data; - for (i = 0; i < dx*dy; i++) { - out_pix[3*i + 0] = back1; - out_pix[3*i + 1] = back2; - out_pix[3*i + 2] = back3; - } - for (j = 0; j < dy/2; j++) { - for (i = 0; i < dx; i++) { - out_pix[j*(extra + 3*dx) + 3*i + 0] = pixel1[10]; - out_pix[j*(extra + 3*dx) + 3*i + 1] = pixel2[10]; - out_pix[j*(extra + 3*dx) + 3*i + 2] = pixel3[10]; - } - - } - */ - image[0].zoom.pix = XCreateImage (graphic[0].display, graphic[0].visual, 24, ZPixmap, 0, - image[0].zoom.data, image[0].zoom.dx, image[0].zoom.dy, 32, 0); - -# endif -} Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/CreateZoom32.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/CreateZoom32.c (revision 25206) +++ (revision ) @@ -1,138 +1,0 @@ -# include "Ximage.h" - -void CreateZoom32 (KapaImageWidget *image, Graphic *graphic, double x, double y) { - - int i, j, ii, jj; - int i_start, i_end, j_start, j_end; - int dropback; /* this is a bit of a kludge... */ - int dx, dy, DX, DY; - double expand, Ix, Iy; - int expand_in, expand_out; - unsigned int *out_pix, *out_pix2; - unsigned short *in_pix, *in_pix2; - unsigned long *pixel, pixvalue; - unsigned long back; - - if (image[0].image[0].matrix.size == 0) { /* create a test pattern */ - REALLOCATE (image[0].zoom.data, char, 4*image[0].zoom.dx*image[0].zoom.dy); - image[0].zoom.pix = XCreateImage (graphic[0].display, graphic[0].visual, graphic[0].depth, ZPixmap, 0, - image[0].zoom.data, image[0].zoom.dx, image[0].zoom.dy, 32, 0); - return; - } - - ALLOCATE (pixel, unsigned long, graphic[0].Npixels); - - // local array for pixel values - for (i = 0; i < graphic[0].Npixels; i++) { - pixel[i] = graphic[0].cmap[i].pixel; - } - back = graphic[0].back; - - // set up expansions - assert ((image[0].zoom.expand >= 1) || (image[0].zoom.expand <= -2)); - expand = expand_in = expand_out = 1.0; - if (image[0].zoom.expand > 0) { - expand = 1 / (1.0*image[0].zoom.expand); - expand_out = image[0].zoom.expand; - expand_in = 1; - } - if (image[0].zoom.expand < 0) { - expand = fabs((double)image[0].zoom.expand); - expand_out = 1; - expand_in = -image[0].zoom.expand; - } - - // define the image sizes - dx = image[0].zoom.dx; - dy = image[0].zoom.dy; - DX = image[0].image[0].matrix.Naxis[0]; - DY = image[0].image[0].matrix.Naxis[1]; - - // x, y are the closest lit screen pixel to 0,0 - Picture_Lower (&i_start, &j_start, &image[0].image[0].matrix, &image[0].zoom); - - // x, y are the closest lit screen pixel to dx, dy - Picture_Upper (&i_end, &j_end, &image[0].image[0].matrix, &image[0].zoom); - - // Ix,Iy are the first displayed image pixel - Picture_to_Image (&Ix, &Iy, i_start, j_start, &image[0].zoom); - - dropback = expand_out - (i_end - i_start) % expand_out; - if ((i_end - i_start) % expand_out == 0) dropback = 0; - - out_pix = (unsigned int *) image[0].zoom.data; - in_pix = &image[0].pixmap[DX*(int)MAX(Iy,0) + (int)MAX(Ix,0)]; - - /********** below we do the mapping from buffer pixels (in) to picture pixels (out) **********/ - - // XXX can we just drop this? - if ((i_end < i_start) || (j_end < j_start)) { - for (j = 0; j < dx*dy; j++, out_pix++) - *out_pix = back; - return; - } - - /**** fill in bottom area ****/ - for (j = 0; j < dx*j_start; j++, out_pix++) { - *out_pix = back; - } - - for (j = j_start; j < j_end; j+= expand_out, out_pix+=(expand_out-1)*dx, in_pix += expand_in*DX) { - - /**** fill in area to the left of the picture ****/ - for (jj = 0; (i_start > 0) && (jj < expand_out); jj++) { - out_pix2 = out_pix + jj*dx; - for (i = 0; i < i_start; i++, out_pix2++) { - *out_pix2 = back; - } - } - out_pix += i_start; - - /*** fill in the picture region ***/ - in_pix2 = in_pix; - if (expand_out == 1) { - for (i = i_start; i < i_end; i++, in_pix2+= expand_in, out_pix++) { - *out_pix = pixel[*in_pix2]; - } - } else { - // equiv to : out_pix += (i_end - i_start) + [expand_out - (i_end - i_start) % expand_out] - for (i = i_start; i < i_end; i+= expand_out, in_pix2++, out_pix+= expand_out) { - pixvalue = pixel[*in_pix2]; - out_pix2 = out_pix; - for (jj = 0; jj < expand_out; jj++, out_pix2+=(dx-expand_out)) { - for (ii = 0; ii < expand_out; ii++, out_pix2++) { - *out_pix2 = pixvalue; - } - } - } - } - out_pix -= dropback; - - /**** fill in area to the right of the picture ****/ - for (jj = 0; jj < expand_out; jj++) { - out_pix2 = out_pix + jj*dx; - for (i = i_end; i < dx; i++, out_pix2++) { - *out_pix2 = back; - } - } - out_pix += (dx - i_end); - assert (out_pix - (unsigned int *)image[0].zoom.data <= dx*dy); - } - assert (out_pix - (unsigned int *)image[0].zoom.data <= dx*dy); - - if ((j_end - j_start) % expand_out > 0) - out_pix -= (expand_out - (j_end - j_start) % expand_out); - - assert (out_pix - (unsigned int *)image[0].zoom.data <= dx*dy); - - /**** fill in top area ****/ - for (j = 0; (j < dx*(dy - j_end)) && (out_pix - (unsigned int *)image[0].zoom.data < dx*dy); j++, out_pix++) { - *out_pix = back; - } - assert (out_pix - (unsigned int *)image[0].zoom.data <= dx*dy); - - image[0].zoom.pix = XCreateImage (graphic[0].display, graphic[0].visual, graphic[0].depth, ZPixmap, 0, - image[0].zoom.data, image[0].zoom.dx, image[0].zoom.dy, 32, 0); - - free (pixel); -} Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/CreateZoom8.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/CreateZoom8.c (revision 25206) +++ (revision ) @@ -1,130 +1,0 @@ -# include "Ximage.h" - -void CreateZoom8 (KapaImageWidget *image, Graphic *graphic, double x, double y) { - -# if (0) - int i, j, ii, jj; - int i_start, i_end, j_start, j_end; - int dropback; /* this is a bit of a kludge... */ - int dx, dy, DX, DY, pixelN; - double expand, zoomscale, Rx, Ry; - int expand_in, expand_out; - unsigned char *out_pix, *out_pix2; - float *imdata, *in_pix, *in_pix2; - unsigned long pixel[256], pixvalue; - unsigned long back; - - if (image[0].image[0].matrix.size == 0) { /* create a test pattern */ - REALLOCATE (image[0].zoom.data, char, image[0].zoom.dx*image[0].zoom.dy); - bzero (image[0].zoom.data, image[0].zoom.dx*image[0].zoom.dy); - image[0].zoom.pix = XCreateImage (graphic[0].display, graphic[0].visual, graphic[0].depth, ZPixmap, 0, - image[0].zoom.data, image[0].zoom.dx, image[0].zoom.dy, 8, 0); - return; - } - - for (i = 0; i < 256; i++) { /* set up pixel array */ - pixel[i] = graphic[0].cmap[i].pixel; - } - back = graphic[0].back; - - // define the color transform parameters - MaxValue = graphic[0].Npixels - 1; - if (image[0].image[0].range != 0.0) { - slope = graphic[0].Npixels / image[0].image[0].range; - start = graphic[0].Npixels * image[0].image[0].zero / image[0].image[0].range; - } else { - slope = 1.0; - start = image[0].image[0].zero; - } - - zoomscale = MAX (5, image[0].expand + 5); - expand = 1 / zoomscale; - expand_out = (int) zoomscale; - expand_in = 1; - - dx = image[0].zoom.dx; - dy = image[0].zoom.dy; - DX = image[0].image[0].matrix.Naxis[0]; - DY = image[0].image[0].matrix.Naxis[1]; - Rx = x - expand*(int)(0.5*(dx + 1)) + 1; - Ry = y - expand*(int)(0.5*(dy + 1)) + 1; - - i_start = MIN (MAX (-Rx / expand, (1 - FRAC(Rx)) / expand), dx - expand_out + 1); - j_start = MIN (MAX (-Ry / expand, (1 - FRAC(Ry)) / expand), dy - expand_out + 1); - i_end = MAX (MIN ((DX-Rx) / expand, dx - expand_out + 1), 0); - j_end = MAX (MIN ((DY-Ry) / expand, dy - expand_out + 1), 0); - dropback = expand_out - (i_end - i_start) % expand_out; - if ((i_end - i_start) % expand_out == 0) dropback = 0; - - out_pix = (unsigned char *) image[0].zoom.data; - imdata = (float *) image[0].image[0].matrix.buffer; - in_pix = &imdata[DX*(int)MAX(Ry,0) + (int)MAX(Rx,0)]; - - /********** below we do the mapping from buffer pixels (in) to picture pixels (out) **********/ - - if ((i_end < i_start) || (j_end < j_start)) { - for (j = 0; j < dx*dy; j++, out_pix++) - *out_pix = back; - return; - } - - /**** fill in bottom area ****/ - for (j = 0; j < dx*j_start; j++, out_pix++) - *out_pix = back; - - for (j = j_start; j < j_end; j+= expand_out, out_pix+=(expand_out-1)*dx, in_pix += expand_in*DX) { - - /**** fill in area to the left of the picture ****/ - for (jj = 0; jj < expand_out; jj++) { - out_pix2 = out_pix + jj*dx; - for (i = 0; i < i_start; i++, out_pix2++) { - *out_pix2 = back; - } - } - out_pix += i_start; - - /*** fill in the picture region ***/ - in_pix2 = in_pix; - if (expand_out == 1) { - for (i = i_start; i < i_end; i++, in_pix2+= expand_in, out_pix++) { - pixelN = PixelLookup(*in_pix2); - *out_pix = pixel[pixelN]; - } - } else { - for (i = i_start; i < i_end; i+= expand_out, in_pix2++, out_pix+= expand_out) { - pixelN = PixelLookup(*in_pix2); - pixvalue = pixel[pixelN]; - out_pix2 = out_pix; - for (jj = 0; jj < expand_out; jj++, out_pix2+=(dx-expand_out)) { - for (ii = 0; ii < expand_out; ii++, out_pix2++) { - *out_pix2 = pixvalue; - } - } - } - } - out_pix -= dropback; - - /**** fill in area to the right of the picture ****/ - for (jj = 0; jj < expand_out; jj++) { - out_pix2 = out_pix + jj*dx; - for (i = i_end; i < dx; i++, out_pix2++) { - *out_pix2 = back; - } - } - out_pix += (dx - i_end); - - } - - if ((j_end - j_start) % expand_out > 0) - out_pix -= expand_out - (j_end - j_start) % expand_out; - - /**** fill in top area ****/ - for (j = 0; (j < dx*(dy - j_end)) && (out_pix - (unsigned char *) image[0].zoom.data < dx*dy); j++, out_pix++) { - *out_pix = back; - } - - image[0].zoom.pix = XCreateImage (graphic[0].display, graphic[0].visual, graphic[0].depth, ZPixmap, 0, - image[0].zoom.data, image[0].zoom.dx, image[0].zoom.dy, 8, 0); - -# endif -} Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/CursorOps.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/CursorOps.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/CursorOps.c (revision 25766) @@ -2,4 +2,5 @@ // input coordinates are relative to the picture bounding box +// XXX pre-calculate fexpand, Xc, Yc when expand is set? void Picture_to_Image (double *x1, double *y1, double x2, double y2, Picture *picture) { @@ -14,10 +15,14 @@ // pixel coordinates in picture frame - dx = expand*(x2 - 0.5*picture[0].dx); - dy = expand*(y2 - 0.5*picture[0].dy); + dx = expand*(x2 - (int)(0.5*picture[0].dx)); + dy = expand*(y2 - (int)(0.5*picture[0].dy)); + + // set the true center (image and screen pixel boundaries must be aligned) + // Xc = (int)(picture[0].Xc / expand) * expand; + // Yc = (int)(picture[0].Yc / expand) * expand; // picture[0].X,Y is the image coordinate in the center of the picture - *x1 = picture[0].flipx ? picture[0].X - dx : picture[0].X + dx; - *y1 = picture[0].flipy ? picture[0].Y - dy : picture[0].Y + dy; + *x1 = picture[0].flipx ? picture[0].Xc - dx : picture[0].Xc + dx; + *y1 = picture[0].flipy ? picture[0].Yc - dy : picture[0].Yc + dy; } @@ -46,10 +51,14 @@ } + // set the true center (image and screen pixel boundaries must be aligned) + // Xc = ((int)(picture[0].Xc * expand)) / expand; + // Yc = ((int)(picture[0].Yc * expand)) / expand; + // pixel coordinates in picture frame - dx = picture[0].flipx ? picture[0].X - x2 : x2 - picture[0].X; - dy = picture[0].flipy ? picture[0].Y - y2 : y2 - picture[0].Y; + dx = picture[0].flipx ? picture[0].Xc - x2 : x2 - picture[0].Xc; + dy = picture[0].flipy ? picture[0].Yc - y2 : y2 - picture[0].Yc; - *x1 = expand*dx + 0.5*picture[0].dx; - *y1 = expand*dy + 0.5*picture[0].dy; + *x1 = expand*dx + (int)(0.5*picture[0].dx); + *y1 = expand*dy + (int)(0.5*picture[0].dy); } @@ -66,5 +75,5 @@ // input coordinates are relative to the picture bounding box -void Picture_Lower (int *i_start, int *j_start, Matrix *matrix, Picture *picture) { +void Picture_Lower (int *i_start, int *j_start, int *I_start, int *J_start, Matrix *matrix, Picture *picture) { double Ix, Iy, Sx, Sy; @@ -72,8 +81,4 @@ // Ix, Iy are the image coordinates of the specified screen pixel Picture_to_Image (&Ix, &Iy, 0.0, 0.0, picture); - - // Ix, Iy are now limited to valid image coordinates - Ix = MIN (MAX (Ix, 0), matrix[0].Naxis[0]); - Iy = MIN (MAX (Iy, 0), matrix[0].Naxis[1]); // round up (down) to nearest pixel boundary @@ -84,4 +89,12 @@ Iy = picture[0].flipy ? (int)(Iy) : (int)(Iy) + 1; } + + // Ix, Iy are now limited to valid image coordinates + Ix = MIN (MAX (Ix, 0), matrix[0].Naxis[0]); + Iy = MIN (MAX (Iy, 0), matrix[0].Naxis[1]); + + // I_start, J_start are the first displayed image pixel + *I_start = Ix; + *J_start = Iy; // Sx, Sy are the screen coordinates of the Ix,Iy pixel @@ -94,6 +107,7 @@ // input coordinates are relative to the picture bounding box -void Picture_Upper (int *i_end, int *j_end, Matrix *matrix, Picture *picture) { +void Picture_Upper (int *i_end, int *j_end, int i_start, int j_start, Matrix *matrix, Picture *picture) { + int nExtra; double Ix, Iy, Sx, Sy; @@ -101,15 +115,19 @@ Picture_to_Image (&Ix, &Iy, picture[0].dx, picture[0].dy, picture); + // round down (up) to nearest pixel boundary + if (Ix > (int)(Ix)) { + Ix = picture[0].flipx ? (int)(Ix) + 1: (int)(Ix); + } + if (Iy > (int)(Iy)) { + Iy = picture[0].flipy ? (int)(Iy) + 1: (int)(Iy); + } + // Ix, Iy are now limited to valid image coordinates Ix = MIN (MAX (Ix, 0), matrix[0].Naxis[0]); Iy = MIN (MAX (Iy, 0), matrix[0].Naxis[1]); - // round down (up) to nearest pixel boundary - if (Ix > (int)(Ix)) { - Ix = picture[0].flipx ? (int)(Ix) + 1 : (int)(Ix); - } - if (Iy > (int)(Iy)) { - Iy = picture[0].flipy ? (int)(Iy) + 1: (int)(Iy); - } + // I_end, J_end are the last displayed image pixel + // *I_end = Ix; + // *J_end = Iy; // Sx, Sy are the screen coordinates of the Ix,Iy pixel @@ -119,3 +137,11 @@ *i_end = MIN (MAX (Sx, 0), picture[0].dx); *j_end = MIN (MAX (Sy, 0), picture[0].dy); + + // round off error can leave us with a small number of extra pixels here. + if (picture[0].expand > 1) { + nExtra = (*i_end - i_start) % picture[0].expand; + *i_end -= nExtra; + nExtra = (*j_end - j_start) % picture[0].expand; + *j_end -= nExtra; + } } Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/DragColorbar.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/DragColorbar.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/DragColorbar.c (revision 25766) @@ -10,5 +10,5 @@ int xstatus, Npix, center; - if (!graphic[0].visualclass) return; + if (!graphic[0].dynamicColors) return; X = mouse_event[0].x; @@ -79,5 +79,5 @@ XColor cmap[256]; - if (!graphic[0].visualclass) return; + if (!graphic[0].dynamicColors) return; for (i = 0; i < graphic[0].Npixels; i++) { @@ -88,4 +88,5 @@ cmap[i].blue = graphic[0].cmap[0].blue; cmap[i].green = graphic[0].cmap[0].green; + cmap[i].flags = DoRed | DoGreen | DoBlue; } else { @@ -94,4 +95,5 @@ cmap[i].blue = graphic[0].cmap[graphic[0].Npixels-1].blue; cmap[i].green = graphic[0].cmap[graphic[0].Npixels-1].green; + cmap[i].flags = DoRed | DoGreen | DoBlue; } else { @@ -99,4 +101,5 @@ cmap[i].blue = graphic[0].cmap[j].blue; cmap[i].green = graphic[0].cmap[j].green; + cmap[i].flags = DoRed | DoGreen | DoBlue; } } Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/Image.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/Image.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/Image.c (revision 25766) @@ -48,18 +48,18 @@ // set the center and expansion for the pictures: - image[0].picture.X = 0.0; - image[0].picture.Y = 0.0; + image[0].picture.Xc = 0.0; + image[0].picture.Yc = 0.0; image[0].picture.expand = 1; image[0].picture.flipx = FALSE; image[0].picture.flipy = FALSE; - image[0].zoom.X = 0.0; - image[0].zoom.Y = 0.0; - image[0].zoom.expand = +5; + image[0].zoom.Xc = 0.0; + image[0].zoom.Yc = 0.0; + image[0].zoom.expand = 5; image[0].zoom.flipx = FALSE; image[0].zoom.flipy = FALSE; - image[0].wide.X = 0.0; - image[0].wide.Y = 0.0; + image[0].wide.Xc = 0.0; + image[0].wide.Yc = 0.0; image[0].wide.expand = -5; image[0].wide.flipx = FALSE; @@ -144,8 +144,10 @@ image[0].picture.dx+1, image[0].picture.dy+1); - XPutImage (graphic[0].display, graphic[0].window, graphic[0].gc, - image[0].picture.pix, 0, 0, - image[0].picture.x + 1, image[0].picture.y + 1, - image[0].picture.dx, image[0].picture.dy); + if (image[0].picture.pix) { + XPutImage (graphic[0].display, graphic[0].window, graphic[0].gc, + image[0].picture.pix, 0, 0, + image[0].picture.x + 1, image[0].picture.y + 1, + image[0].picture.dx, image[0].picture.dy); + } for (i = 0; i < NOVERLAYS; i++) { @@ -156,13 +158,16 @@ if (image[0].location) { - XPutImage (graphic[0].display, graphic[0].window, graphic[0].gc, - image[0].cmapbar.pix, 0, 0, - image[0].cmapbar.x, image[0].cmapbar.y, - image[0].cmapbar.dx, image[0].cmapbar.dy); - - XPutImage (graphic[0].display, graphic[0].window, graphic[0].gc, - image[0].wide.pix, 0, 0, - image[0].wide.x, image[0].wide.y, - image[0].wide.dx, image[0].wide.dy); + if (image[0].cmapbar.pix) { + XPutImage (graphic[0].display, graphic[0].window, graphic[0].gc, + image[0].cmapbar.pix, 0, 0, + image[0].cmapbar.x, image[0].cmapbar.y, + image[0].cmapbar.dx, image[0].cmapbar.dy); + } + if (image[0].wide.pix) { + XPutImage (graphic[0].display, graphic[0].window, graphic[0].gc, + image[0].wide.pix, 0, 0, + image[0].wide.x, image[0].wide.y, + image[0].wide.dx, image[0].wide.dy); + } CrossHairs (graphic, &image[0].zoom); Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/InterpretKeys.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/InterpretKeys.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/InterpretKeys.c (revision 25766) @@ -93,5 +93,5 @@ SetColorScale (graphic, image); Remap (graphic, image); - Reorient (graphic, image, image[0].picture.X, image[0].picture.Y, 0); + Reorient (graphic, image, image[0].picture.Xc, image[0].picture.Yc, 0); break; @@ -100,5 +100,5 @@ SetColorScale (graphic, image); Remap (graphic, image); - Reorient (graphic, image, image[0].picture.X, image[0].picture.Y, 0); + Reorient (graphic, image, image[0].picture.Xc, image[0].picture.Yc, 0); break; @@ -107,5 +107,5 @@ SetColorScale (graphic, image); Remap (graphic, image); - Reorient (graphic, image, image[0].picture.X, image[0].picture.Y, 0); + Reorient (graphic, image, image[0].picture.Xc, image[0].picture.Yc, 0); break; @@ -113,9 +113,11 @@ case XK_Home: image[0].picture.expand = 1; - Reorient (graphic, image, image[0].picture.X, image[0].picture.Y, 0); + image[0].zoom.expand = 5; + Reorient (graphic, image, image[0].picture.Xc, image[0].picture.Yc, 0); break; case XK_KP_End: case XK_End: image[0].picture.expand = 1; + image[0].zoom.expand = 5; Reorient (graphic, image, 0.5*image[0].image[0].matrix.Naxis[0], 0.5*image[0].image[0].matrix.Naxis[1], 0); break; @@ -128,25 +130,25 @@ case XK_Prior: case XK_KP_Prior: - Reorient (graphic, image, image[0].picture.X, image[0].picture.Y, +1); + Reorient (graphic, image, image[0].picture.Xc, image[0].picture.Yc, +1); break; case XK_Next: case XK_KP_Next: - Reorient (graphic, image, image[0].picture.X, image[0].picture.Y, -1); + Reorient (graphic, image, image[0].picture.Xc, image[0].picture.Yc, -1); break; case XK_Up: case XK_KP_Up: - Reorient (graphic, image, image[0].picture.X, image[0].picture.Y + offset, 0); + Reorient (graphic, image, image[0].picture.Xc, image[0].picture.Yc + offset, 0); break; case XK_Down: case XK_KP_Down: - Reorient (graphic, image, image[0].picture.X, image[0].picture.Y - offset, 0); + Reorient (graphic, image, image[0].picture.Xc, image[0].picture.Yc - offset, 0); break; case XK_Left: case XK_KP_Left: - Reorient (graphic, image, image[0].picture.X + offset, image[0].picture.Y, 0); + Reorient (graphic, image, image[0].picture.Xc + offset, image[0].picture.Yc, 0); break; case XK_Right: case XK_KP_Right: - Reorient (graphic, image, image[0].picture.X - offset, image[0].picture.Y, 0); + Reorient (graphic, image, image[0].picture.Xc - offset, image[0].picture.Yc, 0); break; @@ -160,5 +162,5 @@ SetColorScale (graphic, image); Remap (graphic, image); - Reorient (graphic, image, image[0].picture.X, image[0].picture.Y, 0); + Reorient (graphic, image, image[0].picture.Xc, image[0].picture.Yc, 0); break; case XK_KP_Subtract: @@ -171,5 +173,5 @@ SetColorScale (graphic, image); Remap (graphic, image); - Reorient (graphic, image, image[0].picture.X, image[0].picture.Y, 0); + Reorient (graphic, image, image[0].picture.Xc, image[0].picture.Yc, 0); break; Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/JPEGit24.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/JPEGit24.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/JPEGit24.c (revision 25766) @@ -22,4 +22,5 @@ int ii, i, j; int i_start, i_end, j_start, j_end; + int I_start, J_start; int dropback; /* this is a bit of a kludge... */ int dx, dy, DX, DY, inDX, inDY; @@ -95,17 +96,17 @@ DY = image[0].image[0].matrix.Naxis[1]; - // x, y are the closest lit screen pixel to 0,0 - Picture_Lower (&i_start, &j_start, &image[0].image[0].matrix, &image[0].picture); - - // x, y are the closest lit screen pixel to dx, dy - Picture_Upper (&i_end, &j_end, &image[0].image[0].matrix, &image[0].picture); - - if (i_end < i_start) MY_SWAP_INT (i_start, i_end); - if (j_end < j_start) MY_SWAP_INT (j_start, j_end); - - // Ix,Iy are the first displayed image pixel - Picture_to_Image (&Ix, &Iy, i_start, j_start, &image[0].picture); - Ix = MIN (MAX (0, Ix), DX - 1); - Iy = MIN (MAX (0, Iy), DY - 1); + // i_start, j_start are the closest lit screen pixel to 0,0 + // I_start, J_start are the image pixel corresponding to i_start, j_start + Picture_Lower (&i_start, &j_start, &I_start, &J_start, &image[0].image[0].matrix, &image[0].picture); + + // i_end, j_end are the closest lit screen pixel to dx, dy + // I_end, J_end are the image pixel corresponding to i_end, j_end + Picture_Upper (&i_end, &j_end, i_start, j_start, &image[0].image[0].matrix, &image[0].picture); + + assert (i_start <= i_end); + assert (j_start <= j_end); + + Ix = image[0].picture.flipx ? I_start - 1 : I_start; + Iy = image[0].picture.flipy ? J_start - 1 : J_start; inDX = image[0].picture.flipx ? -1 : +1; Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/LoadPicture.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/LoadPicture.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/LoadPicture.c (revision 25766) @@ -25,6 +25,7 @@ Yoffset = 0.0; if (image[0].image[0].matrix.size) { - Xoffset = image[0].picture.X - 0.5*image[0].image[0].matrix.Naxis[0]; - Yoffset = image[0].picture.Y - 0.5*image[0].image[0].matrix.Naxis[1]; + // XXX enforce int center here? + Xoffset = image[0].picture.Xc - 0.5*image[0].image[0].matrix.Naxis[0]; + Yoffset = image[0].picture.Yc - 0.5*image[0].image[0].matrix.Naxis[1]; } @@ -49,6 +50,6 @@ // reference point for image is the center pixel - image[0].picture.X = 0.5*header.Naxis[0] + Xoffset; - image[0].picture.Y = 0.5*header.Naxis[1] + Yoffset; + image[0].picture.Xc = 0.5*header.Naxis[0] + Xoffset; + image[0].picture.Yc = 0.5*header.Naxis[1] + Yoffset; // choose expand for wide to guarantee we fit: @@ -59,6 +60,6 @@ image[0].wide.expand = 1.0 / wx; } - image[0].wide.X = 0.5*header.Naxis[0]; - image[0].wide.Y = 0.5*header.Naxis[1]; + image[0].wide.Xc = 0.5*header.Naxis[0]; + image[0].wide.Yc = 0.5*header.Naxis[1]; fcntl (sock, F_SETFL, O_NONBLOCK); Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/MakeColormap.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/MakeColormap.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/MakeColormap.c (revision 25766) @@ -2,4 +2,5 @@ static char default_cmap[] = "grayscale"; +// static char default_cmap[] = "heat"; void MakeColormap (int argc, char **argv) { Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/PSOverlay.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/PSOverlay.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/PSOverlay.c (revision 25766) @@ -80,11 +80,11 @@ double sn = sin(angle); x0 = X + pX*dX*cs; - y0 = Y + pY*dX*sn; + y0 = Y - pY*dX*sn; // XXX dt should be based on the size of the ellipse... // 0.10 -> 60 segments on the ellipse # define DT 0.1 for (t = DT; t < 2*M_PI + DT; t+=DT) { - x1 = X + pX*dX*cos(t)*cs - pX*dY*sin(t)*sn; - y1 = Y + pY*dX*cos(t)*sn + pY*dY*sin(t)*cs; + x1 = X + pX*dX*cos(t)*cs + pX*dY*sin(t)*sn; + y1 = Y - pY*dX*cos(t)*sn + pY*dY*sin(t)*cs; fprintf (f, " %6.1f %6.1f %6.1f %6.1f L\n", x0 + extra, y0 + extra, x1 + extra, y1 + extra); x0 = x1; Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/PaintOverlay.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/PaintOverlay.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/PaintOverlay.c (revision 25766) @@ -9,5 +9,5 @@ XSetForeground (graphic[0].display, graphic[0].gc, image[0].overlay[N].color); - XSetLineAttributes (graphic->display, graphic->gc, 2, LineSolid, CapNotLast, JoinMiter); + XSetLineAttributes (graphic->display, graphic->gc, 1, LineSolid, CapNotLast, JoinMiter); expand = 1.0; @@ -72,11 +72,11 @@ double sn = sin(angle); x0 = X + pX*dx*cs; - y0 = Y - pY*dx*sn; + y0 = Y + pY*dx*sn; // XXX dt should be based on the size of the ellipse... // 0.10 -> 60 segments on the ellipse # define DT 0.1 for (t = DT; t < 2*M_PI + DT; t+=DT) { - x1 = X + pX*dx*cos(t)*cs + pX*dy*sin(t)*sn; - y1 = Y - pY*dx*cos(t)*sn + pY*dy*sin(t)*cs; + x1 = X + pX*dx*cos(t)*cs - pX*dy*sin(t)*sn; + y1 = Y + pY*dx*cos(t)*sn + pY*dy*sin(t)*cs; XDrawLine (graphic[0].display, graphic[0].window, graphic[0].gc, x0, y0, x1, y1); x0 = x1; Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/Relocate.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/Relocate.c (revision 25766) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/Relocate.c (revision 25766) @@ -0,0 +1,30 @@ +# include "Ximage.h" + +// XXX Should there be a base command + KiiMessage command? +int Relocate (int sock) { + + int i, Nsection; + int x, y, dX, dY; + Graphic *graphic; + Section *section; + + graphic = GetGraphic(); + + KiiScanMessage (sock, "%d %d", &x, &y); + + if (!USE_XWINDOW) return (TRUE); + + dX = DisplayWidth (graphic->display, graphic->screen); + dY = DisplayHeight (graphic->display, graphic->screen); + + // let's not lose the window + x = MAX(0, MIN(x, dX - 10)); + y = MAX(0, MIN(y, dY - 10)); + + XMoveWindow (graphic->display, graphic->window, x, y); + + // if (USE_XWINDOW) XClearWindow (graphic->display, graphic->window); + // Refresh (); + + return (TRUE); +} Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/Remap16.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/Remap16.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/Remap16.c (revision 25766) @@ -1,3 +1,4 @@ # include "Ximage.h" +# define OUT_TYPE unsigned short # define MY_SWAP_BYTE(W) { \ @@ -10,15 +11,21 @@ int i, j, ii, jj; int i_start, i_end, j_start, j_end; - int dropback; + int I_start, J_start; + int dropback, inDX, inDY; int dx, dy, DX, DY; double expand, Ix, Iy; int expand_in, expand_out; - unsigned short *out_pix, *out_pix2, *data; + OUT_TYPE *out_pix, *out_pix2, *data; unsigned short *in_pix, *in_pix2; - unsigned short *pixel, pixvalue; - unsigned short back; + OUT_TYPE *pixel, pixvalue; + OUT_TYPE back; int swap_client, swap_server, swap_bytes; - ALLOCATE (pixel, unsigned short, graphic[0].Npixels); + // just skip if there is no data + if (matrix[0].Naxes == 0) return; + if (matrix[0].Naxis[0] == 0) return; + if (matrix[0].Naxis[1] == 0) return; + + ALLOCATE (pixel, OUT_TYPE, graphic[0].Npixels); # ifdef BYTE_SWAP @@ -38,5 +45,4 @@ back = 0xffff & graphic[0].back; if (swap_bytes) MY_SWAP_BYTE (back); - // XXX not certain this is the correct solution... // set up expansions @@ -60,17 +66,25 @@ DY = matrix[0].Naxis[1]; - // x, y are the closest lit screen pixel to 0,0 - Picture_Lower (&i_start, &j_start, matrix, picture); + // i_start, j_start are the closest lit screen pixel to 0,0 + // I_start, J_start are the image pixel corresponding to i_start, j_start + Picture_Lower (&i_start, &j_start, &I_start, &J_start, matrix, picture); - // x, y are the closest lit screen pixel to dx, dy - Picture_Upper (&i_end, &j_end, matrix, picture); + // i_end, j_end are the closest lit screen pixel to dx, dy + // I_end, J_end are the image pixel corresponding to i_end, j_end + Picture_Upper (&i_end, &j_end, i_start, j_start, matrix, picture); - // Ix,Iy are the first displayed image pixel - Picture_to_Image (&Ix, &Iy, i_start, j_start, picture); + assert (i_start <= i_end); + assert (j_start <= j_end); + + Ix = picture[0].flipx ? I_start - 1 : I_start; + Iy = picture[0].flipy ? J_start - 1 : J_start; + + inDX = picture[0].flipx ? -1 : +1; + inDY = picture[0].flipy ? -1 : +1; dropback = expand_out - (i_end - i_start) % expand_out; if ((i_end - i_start) % expand_out == 0) dropback = 0; - data = out_pix = (unsigned short *) picture[0].data; + out_pix = data = (OUT_TYPE *) picture[0].data; in_pix = &image[0].pixmap[DX*(int)MAX(Iy,0) + (int)MAX(Ix,0)]; @@ -82,5 +96,5 @@ } - for (j = j_start; j < j_end; j+= expand_out, in_pix += expand_in*DX) { + for (j = j_start; j < j_end; j+= expand_out, in_pix += inDY*expand_in*DX) { out_pix = &data[j*dx]; @@ -97,9 +111,9 @@ in_pix2 = in_pix; if (expand_out == 1) { - for (i = i_start; i < i_end; i++, in_pix2+= expand_in, out_pix++) { + for (i = i_start; i < i_end; i++, in_pix2 += inDX*expand_in, out_pix++) { *out_pix = pixel[*in_pix2]; } } else { - for (i = i_start; i < i_end; i+= expand_out, in_pix2++, out_pix+= expand_out) { + for (i = i_start; i < i_end; i+= expand_out, in_pix2 += inDX, out_pix+= expand_out) { pixvalue = pixel[*in_pix2]; out_pix2 = out_pix; @@ -124,5 +138,5 @@ /**** fill in top area ****/ out_pix = &data[j_end*dx]; - for (j = 0; j < (dy - j_end); j++) { + for (j = 0; j < dy - j_end; j++) { for (i = 0; i < dx; i++, out_pix++) { *out_pix = back; Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/Remap24.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/Remap24.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/Remap24.c (revision 25766) @@ -5,5 +5,6 @@ int i, j, ii, jj; int i_start, i_end, j_start, j_end; - int dropback, extra; + int I_start, J_start; + int dropback, extra, inDX, inDY; int dx, dy, DX, DY; double expand, Ix, Iy; @@ -53,12 +54,20 @@ extra = 4 - (dx * 3) % 4; - // x, y are the closest lit screen pixel to 0,0 - Picture_Lower (&i_start, &j_start, matrix, picture); + // i_start, j_start are the closest lit screen pixel to 0,0 + // I_start, J_start are the image pixel corresponding to i_start, j_start + Picture_Lower (&i_start, &j_start, &I_start, &J_start, matrix, picture); - // x, y are the closest lit screen pixel to dx, dy - Picture_Upper (&i_end, &j_end, matrix, picture); + // i_end, j_end are the closest lit screen pixel to dx, dy + // I_end, J_end are the image pixel corresponding to i_end, j_end + Picture_Upper (&i_end, &j_end, i_start, j_start, matrix, picture); - // Ix,Iy are the first displayed image pixel - Picture_to_Image (&Ix, &Iy, i_start, j_start, picture); + assert (i_start <= i_end); + assert (j_start <= j_end); + + Ix = picture[0].flipx ? I_start - 1 : I_start; + Iy = picture[0].flipy ? J_start - 1 : J_start; + + inDX = picture[0].flipx ? -1 : +1; + inDY = picture[0].flipy ? -1 : +1; dropback = expand_out - (i_end - i_start) % expand_out; @@ -80,5 +89,5 @@ } - for (j = j_start; j < j_end; j+= expand_out, in_pix += expand_in*DX) { + for (j = j_start; j < j_end; j+= expand_out, in_pix += inDY*expand_in*DX) { out_pix = &data[j*(3*dx+extra)]; @@ -97,5 +106,5 @@ in_pix2 = in_pix; if (expand_out == 1) { - for (i = i_start; i < i_end; i++, in_pix2+= expand_in, out_pix+=3) { + for (i = i_start; i < i_end; i++, in_pix2+= inDX*expand_in, out_pix+=3) { out_pix[0] = pixel1[*in_pix2]; out_pix[1] = pixel2[*in_pix2]; @@ -103,5 +112,5 @@ } } else { - for (i = i_start; i < i_end; i+= expand_out, in_pix2++, out_pix+= 3*expand_out) { + for (i = i_start; i < i_end; i+= expand_out, in_pix2 += inDX, out_pix+= 3*expand_out) { pixvalue1 = pixel1[*in_pix2]; pixvalue2 = pixel2[*in_pix2]; @@ -121,5 +130,4 @@ /**** fill in area to the right of the picture ****/ for (jj = 0; (jj < expand_out) && (j + jj < dy); jj++) { - // for (jj = 0; jj < expand_out; jj++) { out_pix2 = out_pix + jj*(3*dx+extra); for (i = i_end; i < dx; i++, out_pix2+=3) { Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/Remap32.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/Remap32.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/Remap32.c (revision 25766) @@ -1,11 +1,12 @@ # include "Ximage.h" +# define OUT_TYPE unsigned int # define MY_SWAP_INT(A,B) { int tmp; tmp = A; A = B; B = tmp; } -# define MY_SWAP_WORD(W) { \ - char tmp, *X; \ - X = (char *) &W; \ - tmp = X[0]; X[0] = X[3]; X[3] = tmp; \ - tmp = X[1]; X[1] = X[2]; X[2] = tmp; } +# define MY_SWAP_WORD(W) { \ + char tmp, *X; \ + X = (char *) &W; \ + tmp = X[0]; X[0] = X[3]; X[3] = tmp; \ + tmp = X[1]; X[1] = X[2]; X[2] = tmp; } void Remap32 (Graphic *graphic, KapaImageWidget *image, Picture *picture, Matrix *matrix) { @@ -13,15 +14,21 @@ int i, j, ii, jj; int i_start, i_end, j_start, j_end; + int I_start, J_start; int dropback, inDX, inDY; int dx, dy, DX, DY; double expand, Ix, Iy; int expand_in, expand_out; - unsigned int *out_pix, *out_pix2; + OUT_TYPE *out_pix, *out_pix2, *data; unsigned short *in_pix, *in_pix2; - unsigned long *pixel, pixvalue; - unsigned long back; + OUT_TYPE *pixel, pixvalue; + OUT_TYPE back; int swap_client, swap_server, swap_bytes; - ALLOCATE (pixel, unsigned long, graphic[0].Npixels); + // just skip if there is no data + if (matrix[0].Naxes == 0) return; + if (matrix[0].Naxis[0] == 0) return; + if (matrix[0].Naxis[1] == 0) return; + + ALLOCATE (pixel, OUT_TYPE, graphic[0].Npixels); # ifdef BYTE_SWAP @@ -43,5 +50,5 @@ // set up expansions - assert ((picture[0].expand >= 1) || (picture[0].expand <= -1)); + assert ((picture[0].expand >= 1) || (picture[0].expand <= -2)); expand = expand_in = expand_out = 1.0; if (picture[0].expand > 0) { @@ -62,22 +69,17 @@ DY = matrix[0].Naxis[1]; - // x, y are the closest lit screen pixel to 0,0 - Picture_Lower (&i_start, &j_start, matrix, picture); + // i_start, j_start are the closest lit screen pixel to 0,0 + // I_start, J_start are the image pixel corresponding to i_start, j_start + Picture_Lower (&i_start, &j_start, &I_start, &J_start, matrix, picture); - // x, y are the closest lit screen pixel to dx, dy - Picture_Upper (&i_end, &j_end, matrix, picture); + // i_end, j_end are the closest lit screen pixel to dx, dy + // I_end, J_end are the image pixel corresponding to i_end, j_end + Picture_Upper (&i_end, &j_end, i_start, j_start, matrix, picture); - if (i_end < i_start) MY_SWAP_INT (i_start, i_end); - if (j_end < j_start) MY_SWAP_INT (j_start, j_end); + assert (i_start <= i_end); + assert (j_start <= j_end); - // Ix,Iy are the first displayed image pixel - Picture_to_Image (&Ix, &Iy, i_start, j_start, picture); - Ix = MIN (MAX (0, Ix), DX - 1); - Iy = MIN (MAX (0, Iy), DY - 1); - // XXX not completely consistent with the i_start, i_end range... - - // we need to offset because i_start points to the bottom edge of Ix - // if (picture[0].flipx) Ix -= 1.0; - // if (picture[0].flipy) Iy -= 1.0; + Ix = picture[0].flipx ? I_start - 1 : I_start; + Iy = picture[0].flipy ? J_start - 1 : J_start; inDX = picture[0].flipx ? -1 : +1; @@ -87,5 +89,5 @@ if ((i_end - i_start) % expand_out == 0) dropback = 0; - out_pix = (unsigned int *) picture[0].data; + out_pix = data = (OUT_TYPE *) picture[0].data; in_pix = &image[0].pixmap[DX*(int)MAX(Iy,0) + (int)MAX(Ix,0)]; @@ -97,5 +99,6 @@ } - for (j = j_start; j < j_end; j+= expand_out, out_pix+=(expand_out-1)*dx, in_pix += inDY*expand_in*DX) { + for (j = j_start; j < j_end; j+= expand_out, in_pix += inDY*expand_in*DX) { + out_pix = &data[j*dx]; /**** fill in area to the left of the picture ****/ @@ -126,9 +129,4 @@ } out_pix -= dropback; - - // assert (in_pix2 - image[0].pixmap <= DX*DY); - // assert (in_pix2 - image[0].pixmap >= 0); - // assert (in_pix - image[0].pixmap <= DX*DY); - // assert (in_pix - image[0].pixmap >= 0); /**** fill in area to the right of the picture ****/ @@ -139,22 +137,16 @@ } } - out_pix += (dx - i_end); - // assert (out_pix - (unsigned int *)picture[0].data <= dx*dy); - // assert (out_pix - (unsigned int *)picture[0].data >= 0); } - - if ((j_end - j_start) % expand_out > 0) - out_pix -= expand_out - (j_end - j_start) % expand_out; - - // assert (out_pix - (unsigned int *)picture[0].data <= dx*dy); - // assert (out_pix - (unsigned int *)picture[0].data >= 0); /**** fill in top area ****/ - for (j = 0; (j < dx*(dy - j_end)) && (out_pix - (unsigned int *)picture[0].data < dx*dy); j++, out_pix++) { - *out_pix = back; + out_pix = &data[j_end*dx]; + for (j = 0; j < dy - j_end; j++) { + for (i = 0; i < dx; i++, out_pix ++) { + *out_pix = back; + } } picture[0].pix = XCreateImage (graphic[0].display, graphic[0].visual, graphic[0].depth, ZPixmap, 0, - picture[0].data, picture[0].dx, picture[0].dy, 32, 0); + picture[0].data, picture[0].dx, picture[0].dy, 32, 0); free (pixel); Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/Remap8.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/Remap8.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/Remap8.c (revision 25766) @@ -1,3 +1,4 @@ # include "Ximage.h" +# define OUT_TYPE unsigned char void Remap8 (Graphic *graphic, KapaImageWidget *image, Picture *picture, Matrix *matrix) { @@ -5,14 +6,20 @@ int i, j, ii, jj; int i_start, i_end, j_start, j_end; - int dropback; + int I_start, J_start; + int dropback, inDX, inDY; int dx, dy, DX, DY; double expand, Ix, Iy; int expand_in, expand_out; - unsigned char *out_pix, *out_pix2; - unsigned short *in_pix, *in_pix2; - unsigned char *pixel, pixvalue; - unsigned char back; + OUT_TYPE *out_pix, *out_pix2, *data; + unsigned short *in_pix, *in_pix2; + OUT_TYPE *pixel, pixvalue; + OUT_TYPE back; - ALLOCATE (pixel, unsigned char, graphic[0].Npixels); + // just skip if there is no data + if (matrix[0].Naxes == 0) return; + if (matrix[0].Naxis[0] == 0) return; + if (matrix[0].Naxis[1] == 0) return; + + ALLOCATE (pixel, OUT_TYPE, graphic[0].Npixels); // local array for pixel values @@ -44,17 +51,25 @@ DY = matrix[0].Naxis[1]; - // x, y are the closest lit screen pixel to 0,0 - Picture_Lower (&i_start, &j_start, matrix, picture); + // i_start, j_start are the closest lit screen pixel to 0,0 + // I_start, J_start are the image pixel corresponding to i_start, j_start + Picture_Lower (&i_start, &j_start, &I_start, &J_start, matrix, picture); - // x, y are the closest lit screen pixel to dx, dy - Picture_Upper (&i_end, &j_end, matrix, picture); + // i_end, j_end are the closest lit screen pixel to dx, dy + // I_end, J_end are the image pixel corresponding to i_end, j_end + Picture_Upper (&i_end, &j_end, i_start, j_start, matrix, picture); - // Ix,Iy are the first displayed image pixel - Picture_to_Image (&Ix, &Iy, i_start, j_start, picture); + assert (i_start <= i_end); + assert (j_start <= j_end); + + Ix = picture[0].flipx ? I_start - 1 : I_start; + Iy = picture[0].flipy ? J_start - 1 : J_start; + + inDX = picture[0].flipx ? -1 : +1; + inDY = picture[0].flipy ? -1 : +1; dropback = expand_out - (i_end - i_start) % expand_out; if ((i_end - i_start) % expand_out == 0) dropback = 0; - out_pix = (unsigned char *) picture[0].data; + out_pix = data = (OUT_TYPE *) picture[0].data; in_pix = &image[0].pixmap[DX*(int)MAX(Iy,0) + (int)MAX(Ix,0)]; @@ -66,5 +81,6 @@ } - for (j = j_start; j < j_end; j+= expand_out, out_pix+=(expand_out-1)*dx, in_pix += expand_in*DX) { + for (j = j_start; j < j_end; j+= expand_out, in_pix += inDY*expand_in*DX) { + out_pix = &data[j*dx]; /**** fill in area to the left of the picture ****/ @@ -80,12 +96,12 @@ in_pix2 = in_pix; if (expand_out == 1) { - for (i = i_start; i < i_end; i++, in_pix2+= expand_in, out_pix++) { + for (i = i_start; i < i_end; i++, in_pix2 += inDX*expand_in, out_pix++) { *out_pix = pixel[*in_pix2]; } } else { - for (i = i_start; i < i_end; i+= expand_out, in_pix2++, out_pix+= expand_out) { + for (i = i_start; i < i_end; i+= expand_out, in_pix2 += inDX, out_pix+= expand_out) { pixvalue = pixel[*in_pix2]; out_pix2 = out_pix; - for (jj = 0; (jj < expand_out) & (j + jj < dy); jj++, out_pix2+=(dx-expand_out)) { + for (jj = 0; (jj < expand_out) && (j + jj < dy); jj++, out_pix2+=(dx-expand_out)) { for (ii = 0; ii < expand_out; ii++, out_pix2++) { *out_pix2 = pixvalue; @@ -103,14 +119,12 @@ } } - out_pix += (dx - i_end); - } - if ((j_end - j_start) % expand_out > 0) - out_pix -= expand_out - (j_end - j_start) % expand_out; - /**** fill in top area ****/ - for (j = 0; (j < dx*(dy - j_end)) && (out_pix - (unsigned char *)picture[0].data < dx*dy); j++, out_pix++) { - *out_pix = back; + out_pix = &data[j_end*dx]; + for (j = 0; j < dy - j_end; j++) { + for (i = 0; i < dx; i++, out_pix++) { + *out_pix = back; + } } picture[0].pix = XCreateImage (graphic[0].display, graphic[0].visual, graphic[0].depth, ZPixmap, 0, Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/Reorient.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/Reorient.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/Reorient.c (revision 25766) @@ -7,7 +7,10 @@ picture = &image[0].picture; - if (picture[0].expand == 0) picture[0].expand = 1; + if (picture[0].expand == 0) { + picture[0].expand = 1; + image[0].zoom.expand = 5; + } - if ((picture[0].X == X) && (picture[0].Y == Y) && (mode == 0)) { + if ((picture[0].Xc == X) && (picture[0].Yc == Y) && (mode == 0)) { Refresh (); XFlush (graphic[0].display); @@ -17,7 +20,7 @@ switch (mode) { case 0: - if ((picture[0].X != X) || (picture[0].Y != Y)) { - picture[0].X = X; - picture[0].Y = Y; + if ((picture[0].Xc != X) || (picture[0].Yc != Y)) { + picture[0].Xc = X; + picture[0].Yc = Y; } break; @@ -25,14 +28,15 @@ picture[0].expand--; if ((picture[0].expand == 0) || (picture[0].expand == -1)) picture[0].expand = -2; - picture[0].X = X; - picture[0].Y = Y; + picture[0].Xc = X; + picture[0].Yc = Y; break; case +1: picture[0].expand++; if ((picture[0].expand == 0) || (picture[0].expand == -1)) picture[0].expand = 1; - picture[0].X = X; - picture[0].Y = Y; + picture[0].Xc = X; + picture[0].Yc = Y; break; } + image[0].zoom.expand = MIN(image[0].zoom.dx / 5.5, MAX(5, 2*picture[0].expand)); Remap (graphic, image); Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/SetColorScale.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/SetColorScale.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/SetColorScale.c (revision 25766) @@ -218,5 +218,5 @@ // store the colors if (USE_XWINDOW) { - if (graphic[0].visualclass) { + if (graphic[0].dynamicColors) { XStoreColors(graphic[0].display, graphic[0].colormap, graphic[0].cmap, Npixels); } else { Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/SetColormap.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/SetColormap.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/SetColormap.c (revision 25766) @@ -146,9 +146,11 @@ store_colors: if (!USE_XWINDOW) return (TRUE); - if (graphic[0].visualclass) { - XStoreColors(graphic[0].display, graphic[0].colormap, graphic[0].cmap, graphic[0].Npixels); + if (graphic[0].dynamicColors) { + if (!XStoreColors(graphic[0].display, graphic[0].colormap, graphic[0].cmap, graphic[0].Npixels)) { + fprintf (stderr, "error storing colors\n"); + } } else { for (i = 0; i < graphic[0].Npixels; i++) { - if (XAllocColor (graphic[0].display, graphic[0].colormap, &graphic[0].cmap[i]) == 0) { + if (!XAllocColor (graphic[0].display, graphic[0].colormap, &graphic[0].cmap[i])) { fprintf (stderr, "error on %d\n", i); } Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/SetUpGraphic.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/SetUpGraphic.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/SetUpGraphic.c (revision 25766) @@ -60,4 +60,5 @@ graphic->color = KapaX11colors (graphic->display, graphic->colormap, graphic->fore, &Ncolors); if (MAP_WINDOW) MapWindow (graphic); + return; } Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/StatusBox.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/StatusBox.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/StatusBox.c (revision 25766) @@ -11,10 +11,10 @@ y = 0.5*image[0].image[0].matrix.Naxis[1]; // z = -1; - image[0].zoom.X = x; - image[0].zoom.Y = y; + image[0].zoom.Xc = x; + image[0].zoom.Yc = y; // image[0].z = z; } else { - x = image[0].zoom.X; - y = image[0].zoom.Y; + x = image[0].zoom.Xc; + y = image[0].zoom.Yc; // z = image[0].z; } Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/UpdatePointer.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/UpdatePointer.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/UpdatePointer.c (revision 25766) @@ -29,6 +29,6 @@ skip: - image[0].zoom.X = x; - image[0].zoom.Y = y; + image[0].zoom.Xc = x; + image[0].zoom.Yc = y; UpdateStatusBox (graphic, image, x, y, z, 0); @@ -36,8 +36,10 @@ CreateZoom (graphic, image); - XPutImage (graphic[0].display, graphic[0].window, graphic[0].gc, - image[0].zoom.pix, 0, 0, - image[0].zoom.x, image[0].zoom.y, - image[0].zoom.dx, image[0].zoom.dy); + if (image[0].zoom.pix) { + XPutImage (graphic[0].display, graphic[0].window, graphic[0].gc, + image[0].zoom.pix, 0, 0, + image[0].zoom.x, image[0].zoom.y, + image[0].zoom.dx, image[0].zoom.dy); + } CrossHairs (graphic, &image[0].zoom); XFlush (graphic[0].display); Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/UpdateStatusBox.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/UpdateStatusBox.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/UpdateStatusBox.c (revision 25766) @@ -48,5 +48,5 @@ bzero (line, 100); - sprintf (line, "%10.1f %10.1f", x, y); + sprintf (line, "%10.2f %10.2f", x, y); XDrawString (graphic[0].display, graphic[0].window, graphic[0].gc, image[0].text_x + PAD1, image[0].text_y + 2*textpad + 2*PAD1, line, strlen(line)); Index: branches/eam_branches/20090820/Ohana/src/kapa2/src/bDrawOverlay.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/src/bDrawOverlay.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/kapa2/src/bDrawOverlay.c (revision 25766) @@ -81,11 +81,11 @@ double sn = sin(angle); x0 = X + pX*dx*cs; - y0 = Y - pY*dx*sn; + y0 = Y + pY*dx*sn; // XXX dt should be based on the size of the ellipse... // 0.10 -> 60 segments on the ellipse # define DT 0.1 for (t = DT; t < 2*M_PI + DT; t+=DT) { - x1 = X + pX*dx*cos(t)*cs + pX*dy*sin(t)*sn; - y1 = Y - pY*dx*cos(t)*sn + pY*dy*sin(t)*cs; + x1 = X + pX*dx*cos(t)*cs - pX*dy*sin(t)*sn; + y1 = Y + pY*dx*cos(t)*sn + pY*dy*sin(t)*cs; bDrawLine (x0, y0, x1, y1); x0 = x1; Index: branches/eam_branches/20090820/Ohana/src/kapa2/test/overlay.sh =================================================================== --- branches/eam_branches/20090820/Ohana/src/kapa2/test/overlay.sh (revision 25766) +++ branches/eam_branches/20090820/Ohana/src/kapa2/test/overlay.sh (revision 25766) @@ -0,0 +1,10 @@ + +macro checker + mcreate a 8 8 + for ix 0 8 + for iy 0 8 + if (($ix + $iy) % 2) continue + a[$ix][$iy] = $ix + $iy + end + end +end Index: branches/eam_branches/20090820/Ohana/src/libdvo/doc/locking.txt =================================================================== --- branches/eam_branches/20090820/Ohana/src/libdvo/doc/locking.txt (revision 25766) +++ branches/eam_branches/20090820/Ohana/src/libdvo/doc/locking.txt (revision 25766) @@ -0,0 +1,13 @@ + +The DVO locking policy is a little bit broken. DVO currently +(2009.08.10) requires that users have write access to files because +the code does not distinguish well cases where read-only (soft locks) +are used vs cases of read/write (hard locks) are needed. + +locks set in DVO: + +dvo_catalog_lock -- on catalog open & catalog create (all files for + split mode) + +gfits_db_lock -- on photcode table read/write and dvo_image_lock + Index: branches/eam_branches/20090820/Ohana/src/libdvo/include/dvo.h =================================================================== --- branches/eam_branches/20090820/Ohana/src/libdvo/include/dvo.h (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/libdvo/include/dvo.h (revision 25766) @@ -325,5 +325,5 @@ PhotCodeData *GetPhotcodeTable (void); -int LoadPhotcodes (char *catdir_file, char *master_file); +int LoadPhotcodes (char *catdir_file, char *master_file, int readwrite); int LoadPhotcodesText (char *filename); int LoadPhotcodesFITS (char *filename); @@ -432,5 +432,5 @@ SkyTable *SkyTableLoad PROTO((char *filename, int VERBOSE)); SkyTable *SkyTableFromGSC PROTO((char *filename, int depth, int VERBOSE)); -SkyTable *SkyTableLoadOptimal PROTO((char *catdir, char *SKYFILE, char *GSCFILE, int depth, int VERBOSE)); +SkyTable *SkyTableLoadOptimal PROTO((char *catdir, char *SKYFILE, char *GSCFILE, int readwrite, int depth, int VERBOSE)); int SkyTableSetDepth PROTO((SkyTable *sky, int depth)); SkyList *SkyRegionByPoint PROTO((SkyTable *table, int depth, double ra, double dec)); Index: branches/eam_branches/20090820/Ohana/src/libdvo/src/LoadPhotcodes.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/libdvo/src/LoadPhotcodes.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/libdvo/src/LoadPhotcodes.c (revision 25766) @@ -1,15 +1,20 @@ # include -int LoadPhotcodes (char *catdir_file, char *master_file) { +int LoadPhotcodes (char *catdir_file, char *master_file, int readwrite) { /* first try to load the photcodes from the specified CATDIR location */ if (LoadPhotcodesFITS (catdir_file)) return TRUE; + if (!readwrite) { + fprintf (stderr, "db is missing a photcode table & access is read-only -- create one with photcode-table -import\n"); + return FALSE; + } + /* next try to load the photcodes from the master text photcode file */ /* automatically (or on demand?) save the text file to the FITS version */ if (LoadPhotcodesText (master_file)) { - if (!check_file_access (catdir_file, TRUE, TRUE)) return TRUE; - SavePhotcodesFITS (catdir_file); - return TRUE; + if (!check_file_access (catdir_file, TRUE, TRUE, TRUE)) return TRUE; + SavePhotcodesFITS (catdir_file); + return TRUE; } Index: branches/eam_branches/20090820/Ohana/src/libdvo/src/coordops.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/libdvo/src/coordops.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/libdvo/src/coordops.c (revision 25766) @@ -163,4 +163,5 @@ return (FALSE); } + if (fabs(alpha) >= 180.0) return (FALSE); *ra = alpha + coords[0].crval1; *dec = delta + coords[0].crval2; Index: branches/eam_branches/20090820/Ohana/src/libdvo/src/dvo_catalog.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/libdvo/src/dvo_catalog.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/libdvo/src/dvo_catalog.c (revision 25766) @@ -180,4 +180,5 @@ int Nsecfilt, mode; + int BACKUP, READWRITE; mode = DVO_OPEN_NONE; @@ -192,9 +193,17 @@ dvo_catalog_init (catalog, FALSE); + // default access control options: catalog[0].lockmode = LCK_XCLD; - if (mode == DVO_OPEN_READ) catalog[0].lockmode = LCK_SOFT; + BACKUP = TRUE; + READWRITE = TRUE; + + // in read-only mode, do not backup or require write access + if (mode == DVO_OPEN_READ) { + catalog[0].lockmode = LCK_SOFT; + BACKUP = FALSE; + READWRITE = FALSE; + } - // XXX make a backup? always? - if (!check_file_access (catalog[0].filename, TRUE, VERBOSE)) { + if (!check_file_access (catalog[0].filename, BACKUP, READWRITE, VERBOSE)) { if (VERBOSE) fprintf (stderr, "no permission to access %s\n", catalog[0].filename); return (FALSE); Index: branches/eam_branches/20090820/Ohana/src/libdvo/src/dvo_image.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/libdvo/src/dvo_image.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/libdvo/src/dvo_image.c (revision 25766) @@ -4,6 +4,16 @@ int dvo_image_lock (FITS_DB *db, char *filename, double timeout, int lockstate) { - /* lock the image catalog */ - if (!check_file_access (filename, FALSE, TRUE)) return (FALSE); + int READWRITE; + + // default access control options: + READWRITE = TRUE; + + // in read-only mode, do not backup or require write access + if (lockstate == LCK_SOFT) { + READWRITE = FALSE; + } + + // do not perform a backup here + if (!check_file_access (filename, FALSE, READWRITE, TRUE)) return (FALSE); db[0].lockstate = lockstate; Index: branches/eam_branches/20090820/Ohana/src/libdvo/src/skyregion_io.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/libdvo/src/skyregion_io.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/libdvo/src/skyregion_io.c (revision 25766) @@ -84,5 +84,5 @@ } -SkyTable *SkyTableLoadOptimal (char *catdir, char *skyfile, char *gscfile, int depth, int verbose) { +SkyTable *SkyTableLoadOptimal (char *catdir, char *skyfile, char *gscfile, int readwrite, int depth, int verbose) { char filename[256]; @@ -93,5 +93,5 @@ sprintf (filename, "%s/SkyTable.fits", catdir); if (stat (filename, &filestat)) goto SKYFILE; - if (!check_file_access (filename, FALSE, verbose)) goto SKYFILE; + if (!check_file_access (filename, FALSE, readwrite, verbose)) goto SKYFILE; sky = SkyTableLoad (filename, verbose); if (sky == NULL) { @@ -106,5 +106,5 @@ if (skyfile[0] != 0) goto GSCFILE; if (stat (skyfile, &filestat)) goto GSCFILE; - if (!check_file_access (skyfile, FALSE, verbose)) goto GSCFILE; + if (!check_file_access (skyfile, FALSE, readwrite, verbose)) goto GSCFILE; sky = SkyTableLoad (skyfile, verbose); if (sky == NULL) { @@ -117,5 +117,5 @@ /* create CATDIR copy */ sprintf (filename, "%s/SkyTable.fits", catdir); - check_file_access (filename, FALSE, verbose); + check_file_access (filename, FALSE, readwrite, verbose); if (!SkyTableSave (sky, filename)) return NULL; @@ -134,5 +134,5 @@ /* create CATDIR copy */ sprintf (filename, "%s/SkyTable.fits", catdir); - check_file_access (filename, FALSE, verbose); + check_file_access (filename, FALSE, readwrite, verbose); if (!SkyTableSave (sky, filename)) return NULL; Index: branches/eam_branches/20090820/Ohana/src/libkapa/include/kapa.h =================================================================== --- branches/eam_branches/20090820/Ohana/src/libkapa/include/kapa.h (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/libkapa/include/kapa.h (revision 25766) @@ -154,4 +154,5 @@ /* KapaWindow.c */ int KiiResize (int fd, int Nx, int Ny); +int KiiRelocate (int fd, int x, int y); int KiiCenter (int fd, double x, double y, int zoom); int KiiParity (int fd, int xflip, int yflip); Index: branches/eam_branches/20090820/Ohana/src/libkapa/src/KapaWindow.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/libkapa/src/KapaWindow.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/libkapa/src/KapaWindow.c (revision 25766) @@ -23,4 +23,12 @@ KiiSendCommand (fd, 4, "RSIZ"); KiiSendMessage (fd, "%d %d", Nx, Ny); + KiiWaitAnswer (fd, "DONE"); + return (TRUE); +} + +int KiiRelocate (int fd, int x, int y) { + + KiiSendCommand (fd, 4, "MOVE"); + KiiSendMessage (fd, "%d %d", x, y); KiiWaitAnswer (fd, "DONE"); return (TRUE); Index: branches/eam_branches/20090820/Ohana/src/libohana/include/ohana.h =================================================================== --- branches/eam_branches/20090820/Ohana/src/libohana/include/ohana.h (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/libohana/include/ohana.h (revision 25766) @@ -166,5 +166,5 @@ int mkdirhier PROTO((char *path, int mode)); void make_backup PROTO((char *filename)); -int check_file_access PROTO((char *basefile, int backup, int verbose)); +int check_file_access PROTO((char *basefile, int backup, int readwrite, int verbose)); int check_dir_access PROTO((char *path, int verbose)); int check_file_exec PROTO((char *filename)); Index: branches/eam_branches/20090820/Ohana/src/libohana/src/findexec.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/libohana/src/findexec.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/libohana/src/findexec.c (revision 25766) @@ -67,5 +67,5 @@ - file backup permission OK (optional) */ -int check_file_access (char *basefile, int BACKUP, int VERBOSE) { +int check_file_access (char *basefile, int BACKUP, int READWRITE, int VERBOSE) { char *path, *filename; @@ -74,7 +74,12 @@ gid_t gid; int status; + int valid; uid = getuid(); gid = getgid(); + + // XXX this function needs to call 'getgroups' to get the full list of the user's + // groups. we would then need to loop over all groups in the gid test below + // to see if any match the file. test to see how slow this is. /* check permission to write to directory */ @@ -87,8 +92,18 @@ status = stat (basefile, &filestat); if (status == 0) { /* file exists, are permissions OK? */ - if (((uid == filestat.st_uid) && (filestat.st_mode & S_IRUSR) && (filestat.st_mode & S_IWUSR)) || - ((gid == filestat.st_gid) && (filestat.st_mode & S_IRGRP) && (filestat.st_mode & S_IWGRP)) || - ((filestat.st_mode & S_IROTH) && (filestat.st_mode & S_IWOTH))) { - } else { + valid = FALSE; + if (!valid && (uid == filestat.st_uid)) { + valid = (filestat.st_mode & S_IRUSR) != 0; + valid &= !READWRITE || (filestat.st_mode & S_IWUSR); + } + if (!valid && (gid == filestat.st_gid)) { + valid = (filestat.st_mode & S_IRGRP) != 0; + valid &= !READWRITE || (filestat.st_mode & S_IWGRP); + } + if (!valid) { + valid = (filestat.st_mode & S_IROTH) != 0; + valid &= !READWRITE || (filestat.st_mode & S_IWOTH); + } + if (!valid) { if (VERBOSE) fprintf (stderr, "can't write to %s\n", basefile); return (FALSE); @@ -102,8 +117,18 @@ status = stat (filename, &filestat); if (status == 0) { /* file exists, are permissions OK? */ - if (((uid == filestat.st_uid) && (filestat.st_mode & S_IRUSR) && (filestat.st_mode & S_IWUSR)) || - ((gid == filestat.st_gid) && (filestat.st_mode & S_IRGRP) && (filestat.st_mode & S_IWGRP)) || - ((filestat.st_mode & S_IROTH) && (filestat.st_mode & S_IWOTH))) { - } else { + valid = FALSE; + if (!valid && (uid == filestat.st_uid)) { + valid = (filestat.st_mode & S_IRUSR) != 0; + valid &= !READWRITE || (filestat.st_mode & S_IWUSR); + } + if (!valid && (gid == filestat.st_gid)) { + valid = (filestat.st_mode & S_IRGRP) != 0; + valid &= !READWRITE || (filestat.st_mode & S_IWGRP); + } + if (!valid) { + valid = (filestat.st_mode & S_IROTH) != 0; + valid &= !READWRITE || (filestat.st_mode & S_IWOTH); + } + if (!valid) { if (VERBOSE) fprintf (stderr, "can't write to %s\n", filename); return (FALSE); Index: branches/eam_branches/20090820/Ohana/src/markrock/src/ConfigInit.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/markrock/src/ConfigInit.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/markrock/src/ConfigInit.c (revision 25766) @@ -44,5 +44,5 @@ /* XXX this does not yet write out the master photcode table */ sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile)) { + if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) { fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); exit (1); Index: branches/eam_branches/20090820/Ohana/src/markrock/src/markrock.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/markrock/src/markrock.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/markrock/src/markrock.c (revision 25766) @@ -33,6 +33,6 @@ /* if lockfile exists, program will complain and quit */ - if (!check_file_access (argv[1], TRUE, TRUE)) exit (1); - if (!check_file_access (RockCat, TRUE, TRUE)) exit (1); + if (!check_file_access (argv[1], TRUE, TRUE, TRUE)) exit (1); + if (!check_file_access (RockCat, TRUE, TRUE, TRUE)) exit (1); catalog.filename = argv[1]; Index: branches/eam_branches/20090820/Ohana/src/markstar/src/ConfigInit.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/markstar/src/ConfigInit.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/markstar/src/ConfigInit.c (revision 25766) @@ -54,5 +54,5 @@ /* XXX this does not yet write out the master photcode table */ sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile)) { + if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) { fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); exit (1); Index: branches/eam_branches/20090820/Ohana/src/mosastro/src/getptolemy.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/mosastro/src/getptolemy.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/mosastro/src/getptolemy.c (revision 25766) @@ -19,5 +19,5 @@ /* load regions from GSC table, restrict to patch */ - sky = SkyTableLoadOptimal (CATDIR, NULL, GSCFILE, SKY_DEPTH_HST, VERBOSE); + sky = SkyTableLoadOptimal (CATDIR, NULL, GSCFILE, FALSE, SKY_DEPTH_HST, VERBOSE); SkyTableSetFilenames (sky, CATDIR, "cpt"); skylist = SkyListByPatch (sky, -1, &patch); Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/Makefile =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/Makefile (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/Makefile (revision 25766) @@ -21,4 +21,5 @@ $(SRC)/cgrid.$(ARCH).o \ $(SRC)/coords.$(ARCH).o \ +$(SRC)/cdot.$(ARCH).o \ $(SRC)/cplot.$(ARCH).o \ $(SRC)/csystem.$(ARCH).o \ @@ -36,4 +37,11 @@ $(SRC)/medianmap.$(ARCH).o \ $(SRC)/mkgauss.$(ARCH).o \ +$(SRC)/mksersic.$(ARCH).o \ +$(SRC)/galradius.$(ARCH).o \ +$(SRC)/galradbins.$(ARCH).o \ +$(SRC)/galsectors.$(ARCH).o \ +$(SRC)/galprofiles.$(ARCH).o \ +$(SRC)/elliprofile.$(ARCH).o \ +$(SRC)/petrosian.$(ARCH).o \ $(SRC)/multifit.$(ARCH).o \ $(SRC)/objload.$(ARCH).o \ Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/cdot.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/cdot.c (revision 25766) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/cdot.c (revision 25766) @@ -0,0 +1,38 @@ +# include "data.h" + +int cdot (int argc, char **argv) { + + int kapa, status; + Graphdata graphmode; + float x, y; + double r, d, Rmin, Rmax; + + if (!style_args (&graphmode, &argc, argv, &kapa)) return FALSE; + + if (argc != 3) { + gprint (GP_ERR, "USAGE: dot \n"); + return (FALSE); + } + r = atof(argv[1]); + d = atof(argv[2]); + + Rmin = graphmode.coords.crval1 - 182.0; + Rmax = graphmode.coords.crval1 + 182.0; + // Rmid = 0.5*(Rmin + Rmax); + + /* set point style and errorbar mode (these are NOT sticky) */ + graphmode.style = 2; + graphmode.etype = 0; + + r = ohana_normalize_angle (r); + while (r < Rmin) r += 360.0; + while (r > Rmax) r -= 360.0; + + status = fRD_to_XY (&x, &y, r, d, &graphmode.coords); + + if (!KapaPrepPlot (kapa, 1, &graphmode)) return (FALSE); + KapaPlotVector (kapa, 1, &x, "x"); + KapaPlotVector (kapa, 1, &y, "y"); + + return (TRUE); +} Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/cgrid.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/cgrid.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/cgrid.c (revision 25766) @@ -37,5 +37,5 @@ /* set spacings for RA */ minorRA = minorDEC = 0.1; - range = MIN (fabs(graphmode.xmax-graphmode.xmin), fabs(graphmode.ymax-graphmode.ymin)); + range = MIN (fabs(graphmode.coords.cdelt1*(graphmode.xmax-graphmode.xmin)), fabs(graphmode.coords.cdelt2*(graphmode.ymax-graphmode.ymin))); if (NorthPole || SouthPole) range = 360; lrange = log10(MAX(fabs(range), 1e-30)); @@ -58,6 +58,7 @@ } dR = range / 100.0; + /* set spacings for DEC */ - range = MIN (fabs(graphmode.xmax-graphmode.xmin), fabs(graphmode.ymax-graphmode.ymin)); + range = MIN (fabs(graphmode.coords.cdelt1*(graphmode.xmax-graphmode.xmin)), fabs(graphmode.coords.cdelt2*(graphmode.ymax-graphmode.ymin))); lrange = log10(MAX(fabs(range), 1e-30)); factor = (int) (lrange); Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/elliprofile.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/elliprofile.c (revision 25766) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/elliprofile.c (revision 25766) @@ -0,0 +1,64 @@ +# include "astro.h" + +int elliprofile (int argc, char **argv) { + + int i, ix, iy, Nx, Ny, N, center; + float *in; + double Rmaj, Rmin, phi, alpha, Io, ARatio; + double root1, root2, R, A1, A2, A3; + double Sx, Sy, Sxy; + double x, y, r, f, Xo, Yo; + Buffer *buf; + Vector *rvec, *fvec; + + if (argc != 9) { + gprint (GP_ERR, "USAGE: elliprofile (buffer) (rvec) (fvec) (Xo) (Yo) (Rmaj) (Rmin) (phi)\n"); + return (FALSE); + } + + /* select input / output buffers */ + if ((buf = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); + Nx = buf[0].header.Naxis[0]; + Ny = buf[0].header.Naxis[1]; + + if ((rvec = SelectVector (argv[2], ANYVECTOR, TRUE)) == NULL) return (FALSE); + if ((fvec = SelectVector (argv[3], ANYVECTOR, TRUE)) == NULL) return (FALSE); + + ResetVector (rvec, OPIHI_FLT, Nx*Ny); + ResetVector (fvec, OPIHI_FLT, Nx*Ny); + + Xo = atof(argv[4]); + Yo = atof(argv[5]); + + /* shape parameters */ + Rmaj = atof (argv[6]); + Rmin = atof (argv[7]); + phi = atof (argv[8]); + + /* given Rmaj, Rmin, phi, find Sx, Sy, Sxy */ + root1 = SQ(1.0 / Rmaj); + root2 = SQ(1.0 / Rmin); + + // XXX check this + R = 0.5 * (root1 - root2); + A1 = 0.25*(root1 + root2) - 0.5*R*cos(2*RAD_DEG*phi); + A2 = 0.25*(root1 + root2) + 0.5*R*cos(2*RAD_DEG*phi); + A3 = -R*sin(2*RAD_DEG*phi); + + Sx = 0.5/A1; + Sy = 0.5/A2; + Sxy = A3; + + /* f = exp (-r^alpha), r^2 = (x^2 / 2Sx) + (y^2 / 2Sy) + Sxy*x*y */ + in = (float *) buf[0].matrix.buffer; + for (i = iy = 0; iy < Ny; iy++) { + for (ix = 0; ix < Nx; ix++, in++, i++) { + x = ix + 0.5 - Xo; + y = iy + 0.5 - Yo; + rvec[0].elements.Flt[i] = sqrt(0.5*x*x/Sx + 0.5*y*y/Sy + x*y*Sxy); + fvec[0].elements.Flt[i] = *in; + } + } + rvec[0].Nelements = fvec[0].Nelements = i; + return (TRUE); +} Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/galcontour.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/galcontour.c (revision 25766) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/galcontour.c (revision 25766) @@ -0,0 +1,89 @@ +# include "astro.h" + +XXX probably not a great algorithm... + + +int galcontour (int argc, char **argv) { + + int i, j, Nx, Ny, N, Nsec, NELEMENTS, Npt; + float *in; + double theta, dtheta, x, y, Xo, Yo; + Buffer *buf; + Vector *xvec, *yvec, *tvec; + char name[128]; + + if (argc != 5) { + gprint (GP_ERR, "USAGE: galcontour (buffer) (Xo) (Yo) (flux)\n"); + gprint (GP_ERR, " generate contour points (Cx) (Cy) (Ct)\n"); + return (FALSE); + } + + /* select input / output buffers */ + if ((buf = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); + Nx = buf[0].header.Naxis[0]; + Ny = buf[0].header.Naxis[1]; + + Xo = atof(argv[2]); + Yo = atof(argv[3]); + flux = atof(argv[4]); + + if ((xvec = SelectVector ("Cx", ANYVECTOR, TRUE)) == NULL) return (FALSE); + if ((yvec = SelectVector ("Cy", ANYVECTOR, TRUE)) == NULL) return (FALSE); + if ((tvec = SelectVector ("Ct", ANYVECTOR, TRUE)) == NULL) return (FALSE); + + N = 0; + NELEMENTS = 100; + ResetVector (xvec, OPIHI_FLT, NELEMENTS); + ResetVector (yvec, OPIHI_FLT, NELEMENTS); + ResetVector (tvec, OPIHI_FLT, NELEMENTS); + + in = (float *) buf[0].matrix.buffer; + + // find the flux transition for each row until not found + + // first go up: + for (j = Yo; j < Ny; j++) { + + // first go left + for (i = Xo; i >= 0; i--) { + F = in[i + j*Nx]; + if (F < flux) { + // interpolate to flux value between f[i,j] and f[i+1,j] + } + } + + for (j = 0; j < Ny; j++) { + for (i = 0; i < Nx; i++, in++) { + x = i - Xo; + y = j - Yo; + theta = DEG_RAD*atan2(y, x); + if (theta < -0.5*dtheta) theta += 360.0; // need to allow -0.5dtheta for first bin + if (theta > 360 - 0.5*dtheta) theta -= 360.0; // need to allow -0.5dtheta for first bin + N = (int)(theta / dtheta + 0.5); + if (N < 0) { + fprintf (stderr, "?"); + continue; + } + if (N >= Nsec) { + fprintf (stderr, "!"); + continue; + } + + fvec[N][0].elements.Flt[fvec[N][0].Nelements] = *in; + rvec[N][0].elements.Flt[rvec[N][0].Nelements] = hypot(x, y); + fvec[N][0].Nelements ++; + rvec[N][0].Nelements ++; + + *in = N; + + if (fvec[N][0].Nelements >= NELEMENTS) { + NELEMENTS += 1000; + for (N = 0; N < Nsec; N++) { + REALLOCATE (fvec[N][0].elements.Flt, opihi_flt, NELEMENTS); + REALLOCATE (rvec[N][0].elements.Flt, opihi_flt, NELEMENTS); + } + } + } + } + return (TRUE); +} Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/galprofiles.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/galprofiles.c (revision 25766) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/galprofiles.c (revision 25766) @@ -0,0 +1,187 @@ +# include "astro.h" + +double Interpolate (const double x, const double y, float *in, int Nx, int Ny); + +int galprofiles (int argc, char **argv) { + + int i, j, Nx, Ny, N, Nsec, NELEMENTS; + float *in; + double theta, dtheta, x, y, Xo, Yo, r, Rmax, value; + Buffer *buf; + Vector **fvec, **rvec, *tvec; + char name[128]; + + if (argc != 5) { + gprint (GP_ERR, "USAGE: galprofiles (buffer) (Xo) (Yo) Nsec\n"); + gprint (GP_ERR, " generates Nsec profiles around the circle; the first is centered on 0.0 degrees\n"); + return (FALSE); + } + + /* select input / output buffers */ + if ((buf = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); + Nx = buf[0].header.Naxis[0]; + Ny = buf[0].header.Naxis[1]; + Rmax = 0.5*hypot(Nx, Ny); + + Xo = atof(argv[2]); + Yo = atof(argv[3]); + Nsec = atof(argv[4]); + + dtheta = 360.0 / Nsec; + // sector i ranges from (i-0.5)*dtheta to (i+0.5)*dtheta + // i = theta / dtheta + 0.5; + + ALLOCATE (fvec, Vector *, Nsec); + ALLOCATE (rvec, Vector *, Nsec); + if ((tvec = SelectVector ("theta", ANYVECTOR, TRUE)) == NULL) return (FALSE); + ResetVector (tvec, OPIHI_FLT, Nsec); + + // dereference pointer + in = (float *) buf[0].matrix.buffer; + + for (i = 0; i < Nsec; i++) { + + // generate the output vector names & vectors + sprintf (name, "flux_%d", i); + if ((fvec[i] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) return (FALSE); + sprintf (name, "rad_%d", i); + if ((rvec[i] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) return (FALSE); + + // allocate initial data space + N = 0; + NELEMENTS = 1000; + ResetVector (fvec[i], OPIHI_FLT, NELEMENTS); + ResetVector (rvec[i], OPIHI_FLT, NELEMENTS); + + // angle for this profile + tvec[0].elements.Flt[i] = i*dtheta; + theta = RAD_DEG*i*dtheta; + + // start at Xo,Yo and find the x,y locations for r_i, theta where r_i increments by 1 pixel + for (r = 0; r < Rmax; r++) { + + // XXX Xo,Yo are referenced to pixels with bounds i+0.0, i+1.0 + x = r * cos (theta) + Xo; + y = r * sin (theta) + Yo; + + // value is NAN if we run off the image + value = Interpolate(x, y, in, Nx, Ny); + if (isnan(value)) break; + + rvec[i][0].elements.Flt[N] = r; + fvec[i][0].elements.Flt[N] = value; + N++; + + if (N >= NELEMENTS) { + NELEMENTS += 100; + REALLOCATE (rvec[N][0].elements.Flt, opihi_flt, NELEMENTS); + REALLOCATE (fvec[N][0].elements.Flt, opihi_flt, NELEMENTS); + } + } + fvec[i][0].Nelements = N; + rvec[i][0].Nelements = N; + } + return (TRUE); +} + +// fast & simple API to interpolate to a subpixel position using bilinear interpolation +// x,y in parent image coordinates (pixel centers at 0.5, 0.5) +// stolen from psLib +double Interpolate (const double x, const double y, float *in, int Nx, int Ny) { + + // allow extrapolation a small distance beyond the edge of valid pixels, but no + // further (this allows the nXskip,nYskip boundary areas to be used as well) + float nXedge = 0.125*Nx; + float nYedge = 0.125*Ny; + + if ((x < -nXedge) || (x > Nx + nXedge) || (y < -nYedge) || (y > Ny + nYedge)) { + return NAN; + } + + // limiting cases: Nx == 1 and/or Ny == 1 + + // if we have a single pixel, there is no spatial information + if ((Nx == 1) && (Ny == 1)) { + return in[0]; + } + + // handle edge cases with extrapolation + const int ix = x - 0.5; // index of reference pixel + const int iy = y - 0.5; // index of reference pixel + + // do numCols,Rows first so we are never < 0 + const int Xs = MAX (MIN (ix, Nx - 2), 0); + const int Ys = MAX (MIN (iy, Ny - 2), 0); + + const int Xe = Xs + 1; + const int Ye = Ys + 1; + + // dx,dy range from 0.0 to 1.0 for interpolated pixels, and -0.5 to 1.5 for extrapolation + const double dx = x - 0.5 - Xs; + const double dy = y - 0.5 - Ys; + + const double rx = 1.0 - dx; + const double ry = 1.0 - dy; + + // if Nx == 1, we have no x-dir spatial information + if (Nx == 1) { + double V0 = in[Ys*Nx + Xs]; + double V1 = in[Ye*Nx + Xs]; + + const double value = V0*ry + V1*dy; + return value; + } + + // if Ny == 1, we have no y-dir spatial information + if (Ny == 1) { + double V0 = in[Ys*Nx + Xs]; + double V1 = in[Ys*Nx + Xe]; + + const double value = V0*rx + V1*dx; + return value; + } + + // Vxy + double V00 = in[Ys*Nx + Xs]; + double V01 = in[Ye*Nx + Xs]; + double V10 = in[Ys*Nx + Xe]; + double V11 = in[Ye*Nx + Xe]; + + double value; + + // corners + if ((dx < 0.0) && (dy < 0.0)) { + return V00; + } + if ((dx > 1.0) && (dy < 0.0)) { + return V10; + } + if ((dx < 0.0) && (dy > 1.0)) { + return V01; + } + if ((dx > 1.0) && (dy > 1.0)) { + return V11; + } + + // sides + if (dx < 0.0) { + value = V00*ry + V01*dy; + return value; + } + if (dy < 0.0) { + value = V00*rx + V10*dx; + return value; + } + if (dx > 1.0) { + value = V10*ry + V11*dy; + return value; + } + if (dy > 1.0) { + value = V01*rx + V11*dx; + return value; + } + + // bilinear interpolation + value = V00*rx*ry + V10*dx*ry + V01*rx*dy + V11*dx*dy; + return value; +} Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/galradbins.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/galradbins.c (revision 25766) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/galradbins.c (revision 25766) @@ -0,0 +1,84 @@ +# include "astro.h" + +int MeanSurfaceBrightness (Vector *rvec, Vector *fvec, Vector *Rvec, Vector *Fvec, Vector *Avec, float Rmin, float Rmax, int bin); + +// given a collection of r, f points sampled at pixels, generate a pair of vectors r, f +// where f is defined as the mean surface brightness for \alpha r_i < r < \beta r_i. +// sample r at r_i = i + +// this function can be much more efficient if the input vectors are sorted by R and that +// fact is used in generating the radial bins... +int galradbins (int argc, char **argv) { + + int i, Nbin; + double Rmin, Rmax; + Vector *fvec, *rvec, *Fvec, *Rvec, *Avec; + + if (argc != 6) { + gprint (GP_ERR, "USAGE: galradius (r_in) (f_in) (r_out) (f_out) (area_out)\n"); + return (FALSE); + } + + /* select input / output buffers */ + if ((rvec = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((fvec = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((Rvec = SelectVector (argv[3], ANYVECTOR, TRUE)) == NULL) return (FALSE); + if ((Fvec = SelectVector (argv[4], ANYVECTOR, TRUE)) == NULL) return (FALSE); + if ((Avec = SelectVector (argv[5], ANYVECTOR, TRUE)) == NULL) return (FALSE); + + Rmax = rvec[0].elements.Flt[0]; + for (i = 0; i < rvec[0].Nelements; i++) { + Rmax = MAX(Rmax, rvec[0].elements.Flt[i]); + } + Nbin = 0.8*Rmax + 2; + ResetVector (Rvec, OPIHI_FLT, Nbin); + ResetVector (Fvec, OPIHI_FLT, Nbin); + ResetVector (Avec, OPIHI_FLT, Nbin); + + // the first three bins are specially defined: + + // 0 : r < 1.0 -- Area is pi + MeanSurfaceBrightness (rvec, fvec, Rvec, Fvec, Avec, 0.00, 1.00, 0); + + // area is \int 2 \pi r dr = \pi (r2^2 - r1^2) + + // 1 : 1.0 < r < 1.25 -- Area is \pi (1.25^2 - 1^2) + MeanSurfaceBrightness (rvec, fvec, Rvec, Fvec, Avec, 1.00, 1.25, 1); + + // 2 : 1.25 < r < 1.60 -- Area is \pi (1.6^2 - 1.25^2) + MeanSurfaceBrightness (rvec, fvec, Rvec, Fvec, Avec, 1.25, 1.60, 2); + + // from bin 3 on out, r = i - 1 + for (i = 3; i < Nbin; i++) { + MeanSurfaceBrightness (rvec, fvec, Rvec, Fvec, Avec, 0.8*(i-1), 1.25*(i-1), i); + } + + return (TRUE); +} + +int MeanSurfaceBrightness (Vector *rvec, Vector *fvec, Vector *Rvec, Vector *Fvec, Vector *Avec, float Rmin, float Rmax, int bin) { + + int i, Npts; + double Fsum; + + Fsum = Npts = 0; + + for (i = 0; i < rvec[0].Nelements; i++) { + if (rvec[0].elements.Flt[i] < Rmin) continue; + if (rvec[0].elements.Flt[i] > Rmax) continue; + Fsum += fvec[0].elements.Flt[i]; + Npts ++; + } + if (Rmin > 0.0) { + Rvec[0].elements.Flt[bin] = sqrt(Rmin * Rmax); // XXX what is the correct mean radius? + } else { + Rvec[0].elements.Flt[bin] = 0.5 * Rmax; // XXX what is the correct mean radius? + } + if (Npts > 0) { + Fvec[0].elements.Flt[bin] = Fsum / Npts; // XXX what is the correct mean radius? + } else { + Fvec[0].elements.Flt[bin] = NAN; // XXX what is the correct mean radius? + } + Avec[0].elements.Flt[bin] = M_PI * (SQ(Rmax) - SQ(Rmin)); + return (TRUE); +} Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/galradius.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/galradius.c (revision 25766) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/galradius.c (revision 25766) @@ -0,0 +1,123 @@ +# include "astro.h" + +// given an image and associated galaxy profile, find: +// a) a characteristic flux level that defines a useful isophot +// b) radius at which the profile crosses that isophot +int galradius (int argc, char **argv) { + + int i, j, Nx, Ny, N, Nsec, NELEMENTS, Rbin, Nout, above; + double theta, dtheta, x, y, Xo, Yo, r, Rmax, Rmin, value; + double Fmin, Fmax, dF, Fm, Fp, Fo, Rsum, Rnpt, Ro; + opihi_flt *flux, *radius, *values; + Buffer *buf; + Vector *fvec, *rvec, *tvec, *Fvec, *Rvec; + char name[128]; + + if (argc != 5) { + gprint (GP_ERR, "USAGE: galradius (radius) (flux) (min_flux) (max_flux)\n"); + return (FALSE); + } + + /* select input / output buffers */ + if ((rvec = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((fvec = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + + if ((Rvec = SelectVector ("rg", ANYVECTOR, TRUE)) == NULL) return (FALSE); + if ((Fvec = SelectVector ("fg", ANYVECTOR, TRUE)) == NULL) return (FALSE); + + Fmin = atof(argv[3]); + Fmax = atof(argv[4]); + + // fvec is a noise sample of the galaxy radial profile at points rvec + // rebin fvec into samples defined by the isophot + + // base selections on fluxes defined by the flux range dF + dF = Fmax - Fmin; + + // select all points with flux in the range Fmin + 0.25*dF to Fmin + 0.75*dF + Fm = Fmin + 0.25*dF; + Fp = Fmin + 0.75*dF; + Fo = Fmin + 0.50*dF; + + Rsum = 0; + Rnpt = 0; + for (i = 0; i < fvec[0].Nelements; i++) { + if (fvec[0].elements.Flt[i] < Fm) continue; + if (fvec[0].elements.Flt[i] > Fp) continue; + + Rsum += rvec[0].elements.Flt[i]; + Rnpt ++; + } + + // determine the binned radius values + if (Rnpt == 0) { + Rbin = 1; + } else { + Rbin = MAX(1, 0.5*(Rsum / Rnpt)); + } + + // do not bother rebinning if the bin size is only 2 or less + if (Rbin <= 2) { + Nout = fvec[0].Nelements; + ResetVector (Rvec, OPIHI_FLT, Nout); + ResetVector (Fvec, OPIHI_FLT, Nout); + memcpy (Fvec[0].elements.Flt, fvec[0].elements.Flt, Fvec[0].Nelements*sizeof(opihi_flt)); + memcpy (Rvec[0].elements.Flt, rvec[0].elements.Flt, Rvec[0].Nelements*sizeof(opihi_flt)); + flux = fvec[0].elements.Flt; + radius = rvec[0].elements.Flt; + } else { + // rebin the vectors by Rbin values: + Nout = fvec[0].Nelements / Rbin + 1; + ALLOCATE (values, opihi_flt, fvec[0].Nelements); + + ResetVector (Rvec, OPIHI_FLT, Nout); + ResetVector (Fvec, OPIHI_FLT, Nout); + flux = Fvec[0].elements.Flt; + radius = Rvec[0].elements.Flt; + + for (i = 0; i < Nout; i++) { + radius[i] = (i + 0.5)*Rbin; + Rmin = radius[i] - 0.5*Rbin; + Rmax = radius[i] + 0.5*Rbin; + + N = 0; + for (j = 0; j < fvec[0].Nelements; j++) { + if (rvec[0].elements.Flt[j] < Rmin) continue; + if (rvec[0].elements.Flt[j] > Rmax) continue; + values[N] = fvec[0].elements.Flt[j]; + N++; + } + + // take the median of the values + dsort (values, N); + if (N > 1) { + flux[i] = (N % 2) ? values[(int)(0.5*N)] : 0.5*(values[N/2] + values[N/2 + 1]); + } else { + flux[i] = values[0]; + } + } + free (values); + } + + above = TRUE; + for (i = 0; i < Nout; i++) { + + // find the largest radius that matches the flux transition + if (above && (flux[i] < Fo)) { + if (i == 0) { + // assume Fmax @ R = 0.0 + Ro = radius[i] * (Fo - Fmax) / (flux[i] - Fmax); + } else { + Ro = radius[i-1] + (radius[i] - radius[i-1]) * (Fo - flux[i-1]) / (flux[i] - flux[i-1]); + } + above = FALSE; + } + + if (!above && (flux[i] >= Fo)) { + above = TRUE; + } + } + + set_variable ("G_R50", Ro); + return (TRUE); +} Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/galsectors.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/galsectors.c (revision 25766) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/galsectors.c (revision 25766) @@ -0,0 +1,107 @@ +# include "astro.h" + +int galsectors (int argc, char **argv) { + + int i, j, Nx, Ny, N, Nsec, NELEMENTS, Npt; + float *in; + double theta, dtheta, x, y, Xo, Yo; + Buffer *buf; + Vector **fvec, **rvec, *tvec, *Rv, *Tv, *Fv; + char name[128]; + + if (argc != 5) { + gprint (GP_ERR, "USAGE: galsectors (buffer) (Xo) (Yo) Nsec\n"); + gprint (GP_ERR, " generates Nsec sectors around the circle; the first is centered on 0.0 degrees\n"); + return (FALSE); + } + + /* select input / output buffers */ + if ((buf = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); + Nx = buf[0].header.Naxis[0]; + Ny = buf[0].header.Naxis[1]; + + Xo = atof(argv[2]); + Yo = atof(argv[3]); + Nsec = atof(argv[4]); + + // for a test, generate R, theta vectors for the whole image: + if ((Rv = SelectVector ("Rvec", ANYVECTOR, TRUE)) == NULL) return (FALSE); + if ((Tv = SelectVector ("Tvec", ANYVECTOR, TRUE)) == NULL) return (FALSE); + if ((Fv = SelectVector ("Fvec", ANYVECTOR, TRUE)) == NULL) return (FALSE); + ResetVector (Rv, OPIHI_FLT, Nx*Ny); + ResetVector (Tv, OPIHI_FLT, Nx*Ny); + ResetVector (Fv, OPIHI_FLT, Nx*Ny); + + in = (float *) buf[0].matrix.buffer; + Npt = 0; + for (j = 0; j < Ny; j++) { + for (i = 0; i < Nx; i++, in++) { + x = i - Xo; + y = j - Yo; + Rv[0].elements.Flt[Npt] = hypot(x, y); + Tv[0].elements.Flt[Npt] = DEG_RAD*atan2(y, x); + Fv[0].elements.Flt[Npt] = *in; + Npt ++; + } + } + return (TRUE); + + dtheta = 360.0 / Nsec; + // sector i ranges from (i-0.5)*dtheta to (i+0.5)*dtheta + // i = theta / dtheta + 0.5; + + ALLOCATE (fvec, Vector *, Nsec); + ALLOCATE (rvec, Vector *, Nsec); + if ((tvec = SelectVector ("theta", ANYVECTOR, TRUE)) == NULL) return (FALSE); + ResetVector (tvec, OPIHI_FLT, Nsec); + + // generate the output vector names + NELEMENTS = 1000; + for (i = 0; i < Nsec; i++) { + sprintf (name, "flux_%d", i); + if ((fvec[i] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) return (FALSE); + sprintf (name, "rad_%d", i); + if ((rvec[i] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) return (FALSE); + ResetVector (fvec[i], OPIHI_FLT, NELEMENTS); + ResetVector (rvec[i], OPIHI_FLT, NELEMENTS); + fvec[i][0].Nelements = 0; + rvec[i][0].Nelements = 0; + tvec[0].elements.Flt[i] = i*dtheta; + } + + in = (float *) buf[0].matrix.buffer; + for (j = 0; j < Ny; j++) { + for (i = 0; i < Nx; i++, in++) { + x = i - Xo; + y = j - Yo; + theta = DEG_RAD*atan2(y, x); + if (theta < -0.5*dtheta) theta += 360.0; // need to allow -0.5dtheta for first bin + if (theta > 360 - 0.5*dtheta) theta -= 360.0; // need to allow -0.5dtheta for first bin + N = (int)(theta / dtheta + 0.5); + if (N < 0) { + fprintf (stderr, "?"); + continue; + } + if (N >= Nsec) { + fprintf (stderr, "!"); + continue; + } + + fvec[N][0].elements.Flt[fvec[N][0].Nelements] = *in; + rvec[N][0].elements.Flt[rvec[N][0].Nelements] = hypot(x, y); + fvec[N][0].Nelements ++; + rvec[N][0].Nelements ++; + + *in = N; + + if (fvec[N][0].Nelements >= NELEMENTS) { + NELEMENTS += 1000; + for (N = 0; N < Nsec; N++) { + REALLOCATE (fvec[N][0].elements.Flt, opihi_flt, NELEMENTS); + REALLOCATE (rvec[N][0].elements.Flt, opihi_flt, NELEMENTS); + } + } + } + } + return (TRUE); +} Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/init.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/init.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/init.c (revision 25766) @@ -5,4 +5,5 @@ int cgrid PROTO((int, char **)); int coords PROTO((int, char **)); +int cdot PROTO((int, char **)); int cplot PROTO((int, char **)); int csystem PROTO((int, char **)); @@ -22,5 +23,12 @@ int imsub PROTO((int, char **)); int medianmap PROTO((int, char **)); +int galsectors PROTO((int, char **)); +int galprofiles PROTO((int, char **)); +int galradius PROTO((int, char **)); +int galradbins PROTO((int, char **)); +int elliprofile PROTO((int, char **)); +int petrosian PROTO((int, char **)); int mkgauss PROTO((int, char **)); +int mksersic PROTO((int, char **)); int multifit PROTO((int, char **)); int objload PROTO((int, char **)); @@ -44,4 +52,5 @@ {1, "cgrid", cgrid, "plot sky coordinate grid"}, {1, "coords", coords, "load coordinates for buffer from file"}, + {1, "cdot", cdot, "plot point in sky coordinates"}, {1, "cplot", cplot, "plot vectors in sky coordinates"}, {1, "csystem", csystem, "convert between coordinate systems"}, @@ -61,4 +70,11 @@ {1, "medianmap", medianmap, "small median image"}, {1, "mkgauss", mkgauss, "generate a 2-D gaussian centered in image"}, + {1, "mksersic", mksersic, "generate a 2-D sersic profile"}, + {1, "galsectors", galsectors, "generate radial vectors for sectors of width dtheta"}, + {1, "galprofiles", galprofiles, "generate radial vectors with interpolation along paths"}, + {1, "galradius", galradius, "generate radial vectors with interpolation along paths"}, + {1, "galradbins", galradbins, "generate radial vectors with interpolation along paths"}, + {1, "elliprofile", elliprofile, "generate radial vectors with interpolation along paths"}, + {1, "petrosian", petrosian, "petrosian parameters given radial bins"}, {1, "multifit", multifit, "fit multi-order spectrum"}, {1, "objload", objload, "plot obj data on Ximage "}, Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/mksersic.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/mksersic.c (revision 25766) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/mksersic.c (revision 25766) @@ -0,0 +1,87 @@ +# include "astro.h" + +int mksersic (int argc, char **argv) { + + int i, j, Nx, Ny, N, center; + float *in; + double Rmaj, Rmin, Theta, alpha, Io, ARatio; + double root1, root2, R, A1, A2, A3; + double Sx, Sy, Sxy; + double x, y, r, f, Xo, Yo; + Buffer *buf; + + Xo = Yo = 0; + center = TRUE; + if ((N = get_argument (argc, argv, "-coord"))) { + remove_argument (N, &argc, argv); + Xo = atof (argv[N]); + remove_argument (N, &argc, argv); + Yo = atof (argv[N]); + remove_argument (N, &argc, argv); + center = FALSE; + } + + Theta = 0.0; + if ((N = get_argument (argc, argv, "-angle"))) { + remove_argument (N, &argc, argv); + Theta = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + ARatio = 1.0; + if ((N = get_argument (argc, argv, "-aratio"))) { + remove_argument (N, &argc, argv); + ARatio = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + if (argc != 5) { + gprint (GP_ERR, "USAGE: mksersic (buffer) (Io) (Rmaj) (alpha)\n"); + return (FALSE); + } + + /* select input / output buffers */ + if ((buf = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); + Nx = buf[0].header.Naxis[0]; + Ny = buf[0].header.Naxis[1]; + if (center) { + Xo = 0.5*Nx; + Yo = 0.5*Ny; + } + + Io = atof (argv[2]); + + /* shape parameters */ + Rmaj = atof (argv[3]); + Rmin = Rmaj / ARatio; + + alpha = atof (argv[4]); + + /* given Rmaj, Rmin, Theta, find Sx, Sy, Sxy */ + root1 = SQ(1.0 / Rmaj); + root2 = SQ(1.0 / Rmin); + + // XXX check this + R = 0.5 * (root1 - root2); + A1 = 0.25*(root1 + root2) - 0.5*R*cos(2*RAD_DEG*Theta); + A2 = 0.25*(root1 + root2) + 0.5*R*cos(2*RAD_DEG*Theta); + A3 = -R*sin(2*RAD_DEG*Theta); + + Sx = 0.5/A1; + Sy = 0.5/A2; + Sxy = A3; + + /* f = exp (-r^alpha), r = (x^2 / 2Sx) + (y^2 / 2Sy) + Sxy*x*y */ + in = (float *) buf[0].matrix.buffer; + for (j = 0; j < Ny; j++) { + for (i = 0; i < Nx; i++, in++) { + x = i + 0.5 - Xo; + y = j + 0.5 - Yo; + r = pow((0.5*x*x/Sx + 0.5*y*y/Sy + x*y*Sxy), alpha); + f = Io*exp (-r); + *in += f; + } + } + + return (TRUE); +} Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/petrosian.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/petrosian.c (revision 25766) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/petrosian.c (revision 25766) @@ -0,0 +1,90 @@ +# include "astro.h" + +int petrosian (int argc, char **argv) { + + int i, above; + float *in; + double Fsum, Asum, Area, R_90, rad_90, flux_90; + Vector *rvec, *avec, *fvec, *Rvec, *Fvec, *Svec, *Avec; + + if (argc != 8) { + gprint (GP_ERR, "USAGE: petrosian (radius) (area) (flux) (P_ratio) (P_flux) (P_sb) (P_area)\n"); + return (FALSE); + } + + /* select input vectors */ + if ((rvec = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((avec = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((fvec = SelectVector (argv[3], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((Rvec = SelectVector (argv[4], ANYVECTOR, TRUE)) == NULL) return (FALSE); + if ((Fvec = SelectVector (argv[5], ANYVECTOR, TRUE)) == NULL) return (FALSE); + if ((Svec = SelectVector (argv[6], ANYVECTOR, TRUE)) == NULL) return (FALSE); + if ((Avec = SelectVector (argv[7], ANYVECTOR, TRUE)) == NULL) return (FALSE); + + // difficult work goes into cleaning the input galaxy profile: + // a) extract radial profiles applying 180 degree symmetry requirement + // b) determine the axial ratio and position angle (at r_50) + // c) extract normalized radial profiles applying outlier clipping + // + // the resulting vectors r, f represent the mean surface brightness at a radius r_i, + // measured between \alpha*r_i and \beta*r_i: f(r_i) = \sum_\alpha*r_i^\beta*r_i flux(r) + + // given a clean set of vectors representing the f_i, r_i, determine F_P, R_P such that: + // f(r_i) / \sum_0^r_i f(r) = f_P + + // generate a vector representing f_P(r_i): + + ResetVector (Fvec, OPIHI_FLT, fvec[0].Nelements); + ResetVector (Rvec, OPIHI_FLT, fvec[0].Nelements); + ResetVector (Svec, OPIHI_FLT, fvec[0].Nelements); + ResetVector (Avec, OPIHI_FLT, fvec[0].Nelements); + + above = TRUE; + Fsum = 0.0; + Asum = 0.0; + Area = avec[0].elements.Flt[0]; + for (i = 0; i < fvec[0].Nelements; i++) { + // for nan bins, we keep the area for use with the next valid bin + if (isnan(fvec[0].elements.Flt[i])) { + Area += avec[0].elements.Flt[i]; + continue; + } + Fsum += fvec[0].elements.Flt[i] * Area; + Asum += Area; + if (i+1 < fvec[0].Nelements) { + Area = avec[0].elements.Flt[i+1]; + } + + Rvec[0].elements.Flt[i] = Asum * fvec[0].elements.Flt[i] / Fsum; + Fvec[0].elements.Flt[i] = Fsum; + Svec[0].elements.Flt[i] = Fsum / Asum; + Avec[0].elements.Flt[i] = Asum; + + // anytime we transition below the petrosian ratio R_90, calculate the radius and flux + // we will keep and report the last (largest radius) value + if (above && (Rvec[0].elements.Flt[i] < R_90)) { + // interpolate Rvec between i-1 and i to R_90 to get flux (Fvec) and radius (rvec) + + if (i == 0) { + // assume Fmax @ R = 0.0 + rad_90 = rvec[0].elements.Flt[i] * (R_90 - 1.0) / (Rvec[0].elements.Flt[i] - 1.0); + flux_90 = Fvec[0].elements.Flt[i] * (R_90 - 1.0) / (Rvec[0].elements.Flt[i] - 1.0); + } else { + rad_90 = rvec[0].elements.Flt[i-1] + (rvec[0].elements.Flt[i] - rvec[0].elements.Flt[i-1]) * (R_90 - Rvec[0].elements.Flt[i-1]) / (Rvec[0].elements.Flt[i] - Rvec[0].elements.Flt[i-1]); + flux_90 = Fvec[0].elements.Flt[i-1] + (Fvec[0].elements.Flt[i] - Fvec[0].elements.Flt[i-1]) * (R_90 - Rvec[0].elements.Flt[i-1]) / (Rvec[0].elements.Flt[i] - Rvec[0].elements.Flt[i-1]); + } + + above = FALSE; + } + + // reset on transitions up, but do not re-calculate rad_90, flux_90 + if (!above && (Rvec[0].elements.Flt[i] >= R_90)) { + above = TRUE; + } + } + + set_variable ("P_R90", rad_90); + set_variable ("P_F90", flux_90); + + return (TRUE); +} Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/region.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/region.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.astro/region.c (revision 25766) @@ -6,5 +6,5 @@ double Ra, Dec, Radius; float dx, dy; - int N, kapa; + int N, kapa, NoClear; char *name; Graphdata graphmode; @@ -46,4 +46,10 @@ remove_argument (N, &argc, argv); graphmode.flipnorth = FALSE; + } + + NoClear = FALSE; + if ((N = get_argument (argc, argv, "-no-clear"))) { + remove_argument (N, &argc, argv); + NoClear = TRUE; } @@ -122,10 +128,10 @@ graphmode.coords.cdelt1 = graphmode.coords.cdelt2 = 1.0; - KapaClearSections (kapa); + if (!NoClear) KapaClearSections (kapa); KapaSetLimits (kapa, &graphmode); /* drop this? */ - sprintf (string, "%8.4f %8.4f (%f)", Ra, Dec, Radius); - KapaSendLabel (kapa, string, 2); + // sprintf (string, "%8.4f %8.4f (%f)", Ra, Dec, Radius); + // KapaSendLabel (kapa, string, 2); // XXX is this the right thing to be doing? Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.basic/substr.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.basic/substr.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.basic/substr.c (revision 25766) @@ -16,9 +16,9 @@ // add a range check here if ((N1 < 0) || (N1 >= strlen(argv[1]))) { - gprint (GP_ERR, "ERROR: N1 out of range\n"); + gprint (GP_ERR, "ERROR: start value out of range in substr command\n"); return (FALSE); } if ((N2 < 0) || ((N2+N1) > strlen(argv[1]))) { - gprint (GP_ERR, "ERROR: N2 out of range\n"); + gprint (GP_ERR, "ERROR: end value out of range in substr command\n"); return (FALSE); } Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/Makefile =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/Makefile (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/Makefile (revision 25766) @@ -97,4 +97,5 @@ $(SRC)/rebin.$(ARCH).o \ $(SRC)/resize.$(ARCH).o \ +$(SRC)/relocate.$(ARCH).o \ $(SRC)/roll.$(ARCH).o \ $(SRC)/rotate.$(ARCH).o \ @@ -110,5 +111,5 @@ $(SRC)/style.$(ARCH).o \ $(SRC)/subraster.$(ARCH).o \ -$(SRC)/subset.$(ARCH).o \ +$(SRC)/subset.$(ARCH).o \ $(SRC)/svd.$(ARCH).o \ $(SRC)/swapbytes.$(ARCH).o \ @@ -118,11 +119,13 @@ $(SRC)/tvcolors.$(ARCH).o \ $(SRC)/tvcontour.$(ARCH).o \ -$(SRC)/tvgrid.$(ARCH).o \ +$(SRC)/tvgrid.$(ARCH).o \ $(SRC)/uniq.$(ARCH).o \ -$(SRC)/unsign.$(ARCH).o \ +$(SRC)/unsign.$(ARCH).o \ $(SRC)/vbin.$(ARCH).o \ +$(SRC)/vgroup.$(ARCH).o \ $(SRC)/vclip.$(ARCH).o \ -$(SRC)/vgauss.$(ARCH).o \ -$(SRC)/vmaxwell.$(ARCH).o \ +$(SRC)/vgauss.$(ARCH).o \ +$(SRC)/vellipse.$(ARCH).o \ +$(SRC)/vmaxwell.$(ARCH).o \ $(SRC)/vgrid.$(ARCH).o \ $(SRC)/vload.$(ARCH).o \ @@ -130,5 +133,5 @@ $(SRC)/vpop.$(ARCH).o \ $(SRC)/vroll.$(ARCH).o \ -$(SRC)/vsmooth.$(ARCH).o \ +$(SRC)/vsmooth.$(ARCH).o \ $(SRC)/vstats.$(ARCH).o \ $(SRC)/wd.$(ARCH).o \ Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/init.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/init.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/init.c (revision 25766) @@ -85,4 +85,5 @@ int rebin PROTO((int, char **)); int resize PROTO((int, char **)); +int relocate PROTO((int, char **)); int roll PROTO((int, char **)); int rotate PROTO((int, char **)); @@ -110,8 +111,10 @@ int unsign PROTO((int, char **)); int vbin PROTO((int, char **)); +int vgroup PROTO((int, char **)); int vclip PROTO((int, char **)); int vect_select PROTO((int, char **)); int vgrid PROTO((int, char **)); int vgauss PROTO((int, char **)); +int vellipse PROTO((int, char **)); int vmaxwell PROTO((int, char **)); int vload PROTO((int, char **)); @@ -218,4 +221,5 @@ {1, "rebin", rebin, "rebin image data by factor of N"}, {1, "resize", resize, "set graphics/image window size"}, + {1, "relocate", relocate, "set graphics/image window position"}, {1, "roll", roll, "roll image to new start point"}, {1, "rotate", rotate, "rotate image"}, @@ -243,8 +247,10 @@ {1, "uniq", uniq, "create a uniq vector subset from a vector"}, {1, "unsign", unsign, "toggle the UNSIGN status"}, - {1, "vbin", vbin, "rebin vector data by a fector of N"}, + {1, "vbin", vbin, "rebin vector data by a factor of N"}, + {1, "vgroup", vgroup, "group y vector into bins defined by x vector values"}, {1, "vclip", vclip, "clip values in a vector to be within a range"}, {1, "vectors", list_vectors, "list vectors"}, {1, "vgauss", vgauss, "fit a Gaussian to a vector"}, + {1, "vellipse", vellipse, "fit a Ellipse to a vector pair"}, {1, "vgrid", vgrid, "generate an image from a triplet of vectors"}, {1, "vhistogram", histogram, "generate histogram from vector"}, Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/relocate.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/relocate.c (revision 25766) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/relocate.c (revision 25766) @@ -0,0 +1,33 @@ +# include "data.h" + +int relocate (int argc, char **argv) { + + char *end; + int x, y; + int N, kapa; + char *name; + + /* display source */ + name = NULL; + if ((N = get_argument (argc, argv, "-n"))) { + remove_argument (N, &argc, argv); + name = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (!GetImage (NULL, &kapa, name)) return (FALSE); + FREE (name); + + if (argc != 3) { + gprint (GP_ERR, "USAGE: relocate x y [-n]\n"); + return (FALSE); + } + + /* x & y are pixels for the screen -- this command has no meaning for non-X */ + + x = atoi (argv[1]); + y = atoi (argv[2]); + + KiiRelocate (kapa, x, y); + return (TRUE); +} + Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/rotate.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/rotate.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/rotate.c (revision 25766) @@ -287,5 +287,5 @@ if (Y >= NY - 1) continue; - c = &in_buff[X * NX*Y]; + c = &in_buff[X + NX*Y]; fx = x - X; fy = y - Y; Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/subset.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/subset.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/subset.c (revision 25766) @@ -6,5 +6,5 @@ char *out; - int i, Npts, size; + int i, Npts, size, valid; Vector *ivec, *ovec, *tvec; @@ -12,6 +12,10 @@ ivec = ovec = tvec = NULL; - if ((argc < 6) || strcmp(argv[2], "=") || strcmp (argv[4], "if")) { - gprint (GP_ERR, "SYNTAX: subset vec = vec if (logic expression)\n"); + valid = TRUE; + valid &= (argc >= 6); + valid &= !strcmp(argv[2], "="); + valid &= !strcmp(argv[4], "if") || !strcmp (argv[4], "where"); + if (!valid) { + gprint (GP_ERR, "SYNTAX: subset vec = vec [if/where] (logic expression)\n"); return (FALSE); } @@ -76,5 +80,5 @@ return (TRUE); - error: +error: DeleteVector (tvec); DeleteVector (ovec); Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/tvcolors.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/tvcolors.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/tvcolors.c (revision 25766) @@ -19,4 +19,5 @@ if (argc != 2) { gprint (GP_ERR, "USAGE: tvcolors (colormap)\n"); + gprint (GP_ERR, " colormap options : greyscale, -greyscale, rainbow, heat, fullcolor, ruffcolor (also grayscale, -grayscale)\n"); return (FALSE); } Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/vellipse.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/vellipse.c (revision 25766) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/vellipse.c (revision 25766) @@ -0,0 +1,272 @@ +# include "data.h" + +/* local private functions */ +opihi_flt fellipseOD (opihi_flt theta, opihi_flt *par, int Npar, opihi_flt *dpar); + +// enum {PAR_X0, PAR_Y0, PAR_RX, PAR_RY, PAR_T0, PAR_P0}; +enum {PAR_RMIN, PAR_EPSILON, PAR_PHI}; + +# define GET_VAR(V,A) { \ + char *c; \ + c = get_variable (A); \ + if (c == NULL) { \ + gprint (GP_ERR, "missing fit parameter A\n"); \ + return (FALSE); \ + } \ + V = atof (c); \ + free (c); } + +int vellipse (int argc, char **argv) { + + int i, N, Npts, Npar, Quiet; + opihi_flt par[6], *pos, *dpos, *theta, **covar; + opihi_flt chisq, ochisq, dchisq; + // opihi_flt Rmaj; + opihi_flt Xmin, Xmax, Ymin, Ymax; + Vector *xvec, *yvec, *Xvec, *Yvec, *tvec; + + Quiet = FALSE; + if ((N = get_argument (argc, argv, "-q"))) { + Quiet = TRUE; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-quiet"))) { + Quiet = TRUE; + remove_argument (N, &argc, argv); + } + + if (argc != 6) { + gprint (GP_ERR, "USAGE: vellipse (theta)\n"); + // gprint (GP_ERR, " uses guesses: E_X0, E_Y0, E_RMAJ, E_RMIN, E_THETA\n"); + gprint (GP_ERR, " uses guesses: E_RMAJ, E_RMIN, E_PHI\n"); + return (FALSE); + } + + if ((xvec = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((yvec = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((Xvec = SelectVector (argv[3], ANYVECTOR, TRUE)) == NULL) return (FALSE); + if ((Yvec = SelectVector (argv[4], ANYVECTOR, TRUE)) == NULL) return (FALSE); + if ((tvec = SelectVector (argv[5], OLDVECTOR, TRUE)) == NULL) return (FALSE); + + REQUIRE_VECTOR_FLT (xvec, FALSE); + REQUIRE_VECTOR_FLT (yvec, FALSE); + + // GET_VAR (par[PAR_X0], "E_X0"); + // GET_VAR (par[PAR_Y0], "E_Y0"); + // GET_VAR (par[PAR_RX], "E_RMAJ"); + // GET_VAR (par[PAR_RY], "E_RMIN"); + // GET_VAR (par[PAR_T0], "E_THETA"); + // GET_VAR (par[PAR_P0], "E_PHI"); + // Npar = 6; + + // we need to generate a vector of alternating x,y values and the independent variable theta: + Xmax = Ymax = Xmin = Ymin = 0.0; + ALLOCATE (pos, opihi_flt, 2*xvec[0].Nelements); + ALLOCATE (dpos, opihi_flt, 2*xvec[0].Nelements); + ALLOCATE (theta, opihi_flt, 2*xvec[0].Nelements); + for (i = 0; i < xvec[0].Nelements; i++) { + pos[2*i+0] = xvec[0].elements.Flt[i]; + pos[2*i+1] = yvec[0].elements.Flt[i]; + Xmin = MIN (Xmin, xvec[0].elements.Flt[i]); + Xmax = MAX (Xmax, xvec[0].elements.Flt[i]); + Ymin = MIN (Ymin, yvec[0].elements.Flt[i]); + Ymax = MAX (Ymax, yvec[0].elements.Flt[i]); + dpos[2*i+0] = 1.0; + dpos[2*i+1] = 1.0; + theta[2*i+0] = atan2(yvec[0].elements.Flt[i], xvec[0].elements.Flt[i]); + theta[2*i+1] = theta[2*i+0]; + // fprintf (stderr, "x,y: %f, %f -> %f, %f\n", yvec[0].elements.Flt[i], xvec[0].elements.Flt[i], yvec[0].elements.Flt[i] - par[PAR_Y0], xvec[0].elements.Flt[i] - par[PAR_X0]); + // fprintf (stderr, "x,y: %f, %f -> %f, %f\n", yvec[0].elements.Flt[i], xvec[0].elements.Flt[i], yvec[0].elements.Flt[i] - par[PAR_Y0], xvec[0].elements.Flt[i] - par[PAR_X0]); + // fprintf (stderr, "theta: %f - %f : %f\n", DEG_RAD*theta[2*i+0], tvec[0].elements.Flt[i], DEG_RAD*theta[2*i+0] - tvec[0].elements.Flt[i]); + } + Npts = 2*xvec[0].Nelements; + + // basic guess from range, PHI in degrees, converted to radians + // GET_VAR (Rmaj, "E_RMAJ"); + // GET_VAR (par[PAR_RMIN], "E_RMIN"); + // GET_VAR (par[PAR_PHI], "E_PHI"); + par[PAR_PHI] = 0.0; + par[PAR_EPSILON] = 0.5; + par[PAR_RMIN] = 0.25*((Xmax - Xmin) + (Ymax - Ymin)); + Npar = 3; + + ochisq = mrqinit (theta, pos, dpos, Npts, par, Npar, fellipseOD, !Quiet); + dchisq = ochisq + 2*Npts; + chisq = ochisq + dchisq; + + for (i = 0; (i < 20) && (chisq > 1e-3) && ((dchisq > 0.001*(Npts - Npar)) || (dchisq <= 0.0)); i++) { + chisq = mrqmin (theta, pos, dpos, Npts, par, Npar, fellipseOD, !Quiet); + dchisq = ochisq - chisq; + ochisq = chisq; + if (!Quiet) gprint (GP_ERR, "dchisq: %f, Ndof: %d\n", dchisq, Npts - Npar); + } + if (!Quiet) gprint (GP_ERR, "%d iterations\n", i); + + ResetVector (Xvec, OPIHI_FLT, xvec[0].Nelements); + ResetVector (Yvec, OPIHI_FLT, xvec[0].Nelements); + for (i = 0; i < xvec[0].Nelements; i++) { + Xvec[0].elements.Flt[i] = fellipseOD (theta[2*i+0], par, Npar, dpos); + Yvec[0].elements.Flt[i] = fellipseOD (theta[2*i+1], par, Npar, dpos); + } + Xvec[0].Nelements = xvec[0].Nelements; + Yvec[0].Nelements = xvec[0].Nelements; + + covar = mrqcovar (Npar); + //set_variable ("E_X0", par[PAR_X0]); + //set_variable ("E_Y0", par[PAR_Y0]); + //set_variable ("E_RMAJ", par[PAR_RX]); + //set_variable ("E_RMIN", par[PAR_RY]); + //set_variable ("E_T0", par[PAR_T0]); + //set_variable ("E_P0", par[PAR_P0]); + + set_variable ("E_RMAJ", par[PAR_RMIN] / par[PAR_EPSILON]); + set_variable ("E_RMIN", par[PAR_RMIN]); + set_variable ("E_PHI", DEG_RAD*par[PAR_PHI]); + + // set_variable ("dE_X0", sqrt(covar[PAR_X0][PAR_X0])); + // set_variable ("dE_Y0", sqrt(covar[PAR_Y0][PAR_Y0])); + // set_variable ("dE_RMAJ", sqrt(covar[PAR_RX][PAR_RX])); + // set_variable ("dE_RMIN", sqrt(covar[PAR_RY][PAR_RY])); + // set_variable ("dE_T0", sqrt(covar[PAR_T0][PAR_T0])); + // set_variable ("dE_P0", sqrt(covar[PAR_P0][PAR_P0])); + + set_variable ("dE_RMAJ", sqrt(covar[PAR_EPSILON][PAR_EPSILON]*covar[PAR_RMIN][PAR_RMIN])); + set_variable ("dE_RMIN", sqrt(covar[PAR_RMIN][PAR_RMIN])); + set_variable ("dE_PHI", DEG_RAD*sqrt(covar[PAR_PHI][PAR_PHI])); + + // if (!Quiet) gprint (GP_ERR, "Xo : %f +/- %f\n", par[PAR_X0], sqrt(covar[PAR_X0][PAR_X0])); + // if (!Quiet) gprint (GP_ERR, "Yo : %f +/- %f\n", par[PAR_Y0], sqrt(covar[PAR_Y0][PAR_Y0])); + // if (!Quiet) gprint (GP_ERR, "RX : %f +/- %f\n", par[PAR_RX], sqrt(covar[PAR_RX][PAR_RX])); + // if (!Quiet) gprint (GP_ERR, "RY : %f +/- %f\n", par[PAR_RY], sqrt(covar[PAR_RY][PAR_RY])); + // if (!Quiet) gprint (GP_ERR, "To : %f +/- %f\n", par[PAR_T0], sqrt(covar[PAR_T0][PAR_T0])); + // if (!Quiet) gprint (GP_ERR, "Po : %f +/- %f\n", par[PAR_P0], sqrt(covar[PAR_P0][PAR_P0])); + + if (!Quiet) gprint (GP_ERR, "RMAJ : %f +/- %f\n", par[PAR_RMIN] / par[PAR_EPSILON], 0.0); + if (!Quiet) gprint (GP_ERR, "RMIN : %f +/- %f\n", par[PAR_RMIN], sqrt(covar[PAR_RMIN][PAR_RMIN])); + if (!Quiet) gprint (GP_ERR, "PHI : %f +/- %f\n", DEG_RAD*par[PAR_PHI], sqrt(covar[PAR_PHI][PAR_PHI])); + + free (pos); + free (dpos); + free (theta); + + mrqfree (Npar); + return (TRUE); +} + +# if (0) +/** + * the full chisq is built of two associated sums over coordinates: + * chisq = sum ((X_obs - X_fit(t))^2 + (Y_obs - Y_fit(t))^2) + * + * the independent variable is Theta: + * X_fit = X0 + RX*cos(theta)*cos(phi) - RY*sin(theta)*sin(phi) + * Y_fit = Y0 + RX*cos(theta)*sin(phi) + RY*sin(theta)*cos(phi) + * + * alternating calls to fellipseOD refer alternatively to X or Y + * + * parameters: Xo, Yo (center), RX, RY (axes), To (angle), Po (phase) + */ + +opihi_flt fellipseOD (opihi_flt theta, opihi_flt *par, int Npar, opihi_flt *dpar) { + + static int pass = 0; + + opihi_flt csphi = cos(par[PAR_P0]); + opihi_flt snphi = sin(par[PAR_P0]); + + opihi_flt dtheta = theta - par[PAR_T0]; + opihi_flt cstht = cos(dtheta); + opihi_flt sntht = sin(dtheta); + + // value is X + if (pass == 0) { + pass = 1; + + opihi_flt value = par[PAR_X0] + par[PAR_RX]*cstht*csphi - par[PAR_RY]*sntht*snphi; + + dpar[PAR_X0] = 1.0; + dpar[PAR_Y0] = 0.0; + dpar[PAR_RX] = +cstht*csphi; + dpar[PAR_RY] = -sntht*snphi; + dpar[PAR_P0] = -par[PAR_RX]*cstht*snphi - par[PAR_RY]*sntht*csphi; + dpar[PAR_T0] = -par[PAR_RX]*sntht*csphi - par[PAR_RY]*cstht*snphi; + + return (value); + } + + // value is Y + if (pass == 1) { + pass = 0; + + opihi_flt value = par[PAR_Y0] + par[PAR_RX]*cstht*snphi + par[PAR_RY]*sntht*csphi; + + dpar[PAR_X0] = 0.0; + dpar[PAR_Y0] = 1.0; + dpar[PAR_RX] = +cstht*snphi; + dpar[PAR_RY] = +sntht*csphi; + dpar[PAR_P0] = +par[PAR_RX]*cstht*csphi - par[PAR_RY]*sntht*snphi; + dpar[PAR_T0] = -par[PAR_RX]*sntht*snphi + par[PAR_RY]*cstht*csphi; + return (value); + } + abort(); +} + +# endif + +/** + * the full chisq is built of two associated sums over coordinates: + * chisq = sum ((X_obs - X_fit(t))^2 + (Y_obs - Y_fit(t))^2) + * + * the independent variable is theta (angle from 0,0 to point): + * X_fit = RMIN*R*cos(theta) + * Y_fit = RMIN*R*sin(theta) + * R = 1 / sqrt(sin^2(theta-phi) + epsilon^2 * cos^2(theta-phi)) + * + * alternating calls to fellipseOD refer alternatively to X or Y + * + * RMIN, EPSILON, PHI + */ + +opihi_flt fellipseOD (opihi_flt alpha, opihi_flt *par, int Npar, opihi_flt *dpar) { + + static int pass = 0; + + opihi_flt cs_alpha = cos(alpha); + opihi_flt sn_alpha = sin(alpha); + + opihi_flt cs_phi = cos(alpha - par[PAR_PHI]); + opihi_flt sn_phi = sin(alpha - par[PAR_PHI]); + + opihi_flt r = 1.0 / sqrt(SQ(sn_phi) + SQ(par[PAR_EPSILON]*cs_phi)); + opihi_flt r3 = pow(r, 3.0); + opihi_flt drdE = -0.5 * r3 * SQ(cs_phi) * 2.0 * par[PAR_EPSILON]; + opihi_flt drdP = -0.5 * r3 * (SQ(par[PAR_EPSILON]) - 1) * 2.0 * cs_phi * sn_phi; + + // value is X + if (pass == 0) { + pass = 1; + + opihi_flt value = par[PAR_RMIN]*cs_alpha*r; + + dpar[PAR_RMIN] = r*cs_alpha; + dpar[PAR_EPSILON] = par[PAR_RMIN]*cs_alpha*drdE; + dpar[PAR_PHI] = 4.0*par[PAR_RMIN]*cs_alpha*drdP; + + return (value); + } + + // value is Y + if (pass == 1) { + pass = 0; + + opihi_flt value = par[PAR_RMIN]*sn_alpha*r; + + dpar[PAR_RMIN] = r*sn_alpha; + dpar[PAR_EPSILON] = par[PAR_RMIN]*sn_alpha*drdE; + dpar[PAR_PHI] = 4.0*par[PAR_RMIN]*sn_alpha*drdP; + + return (value); + } + abort(); +} + Index: branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/vgroup.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/vgroup.c (revision 25766) +++ branches/eam_branches/20090820/Ohana/src/opihi/cmd.data/vgroup.c (revision 25766) @@ -0,0 +1,71 @@ +# include "data.h" + +int vgroup (int argc, char **argv) { + + int i, j, N; + Vector *xin, *yin, *xout, *yout; + opihi_flt xmin, xmax, sum; + double *values; + + // Normalize = FALSE; + // if ((N = get_argument (argc, argv, "-norm"))) { + // remove_argument (N, &argc, argv); + // Normalize = TRUE; + // } + // + // Ignore = FALSE; + // IgnoreValue = 0.0; + // if ((N = get_argument (argc, argv, "-ignore"))) { + // Ignore = TRUE; + // remove_argument (N, &argc, argv); + // IgnoreValue = atof (argv[N]); + // remove_argument (N, &argc, argv); + // } + + if (argc != 5) { + gprint (GP_ERR, "USAGE: vbin \n"); + gprint (GP_ERR, " group x,y values in bins defined by \n"); + return (FALSE); + } + + if ((xin = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((yin = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((xout = SelectVector (argv[3], OLDVECTOR, TRUE)) == NULL) return (FALSE); + if ((yout = SelectVector (argv[4], ANYVECTOR, TRUE)) == NULL) return (FALSE); + + // re-binning creates a float vector + ResetVector (yout, OPIHI_FLT, xout[0].Nelements); + + // storage vector + ALLOCATE (values, double, xin[0].Nelements); + + for (i = 0; i < xout[0].Nelements - 1; i++) { + xmin = xout[0].elements.Flt[i]; + xmax = xout[0].elements.Flt[i+1]; + + N = 0; + for (j = 0; j < xin[0].Nelements; j++) { + if (xin[0].elements.Flt[j] < xmin) continue; + if (xin[0].elements.Flt[j] > xmax) continue; + values[N] = yin[0].elements.Flt[j]; + N++; + } + + dsort (values, N); + if (N > 1) { + sum = (N % 2) ? values[(int)(0.5*N)] : 0.5*(values[N/2] + values[N/2 + 1]); + } else { + sum = values[0]; + } + + // measure the stat for this bin + // sum = 0.0; + // for (j = 0; j < N; j++) { + // sum += values[j]; + // } + // sum /= N; + yout[0].elements.Flt[i] = sum; + } + free (values); + return (TRUE); +} Index: branches/eam_branches/20090820/Ohana/src/opihi/dvo/Makefile =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/dvo/Makefile (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/dvo/Makefile (revision 25766) @@ -73,5 +73,4 @@ $(SRC)/imrough.$(ARCH).o \ $(SRC)/imsearch.$(ARCH).o \ -$(SRC)/imstats.$(ARCH).o \ $(SRC)/lcat.$(ARCH).o \ $(SRC)/lcurve.$(ARCH).o \ Index: branches/eam_branches/20090820/Ohana/src/opihi/dvo/gimages.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/dvo/gimages.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/dvo/gimages.c (revision 25766) @@ -124,5 +124,7 @@ } + // find coordinates of image center XY_to_RD (&Ro, &Do, Xo, Yo, &image[i].coords); + if (fabs(Ro - Ra) > 120.0) continue; local.crval1 = Ro; @@ -138,35 +140,22 @@ Xs = -0.5*image[i].NX; Ys = -0.5*image[i].NY; - Xe = +0.5*image[i].NX; - Ye = +0.5*image[i].NY; } else { Xs = 0.0; Ys = 0.0; - Xe = image[i].NX; - Ye = image[i].NY; } + // find coordinates of an image corner XY_to_RD (&Ro, &Do, Xs, Ys, &image[i].coords); - RD_to_XY (&Xo, &Xo, Ro, Do, &local); + + // find radius of image in arcsec + RD_to_XY (&Xo, &Yo, Ro, Do, &local); Radius = hypot (Xo, Yo); // fprintf (stderr, "%s: %f %f %f ", image[i].name, local.crval1, local.crval2, Radius); - XY_to_RD (&Ro, &Do, Xs, Ye, &image[i].coords); - RD_to_XY (&Xo, &Xo, Ro, Do, &local); - Radius = MAX (Radius, hypot (Xo, Yo)); - // fprintf (stderr, "%f ", Radius); - - XY_to_RD (&Ro, &Do, Xe, Ys, &image[i].coords); - RD_to_XY (&Xo, &Xo, Ro, Do, &local); - Radius = MAX (Radius, hypot (Xo, Yo)); - // fprintf (stderr, "%f ", Radius); - - XY_to_RD (&Ro, &Do, Xe, Ye, &image[i].coords); - RD_to_XY (&Xo, &Xo, Ro, Do, &local); - Radius = MAX (Radius, hypot (Xo, Yo)); - // fprintf (stderr, "%f ", Radius); - + // check for distances to coordinates in arcsec RD_to_XY (&Xo, &Yo, Ra, Dec, &local); // fprintf (stderr, " : %f\n", hypot(Xo,Yo)); + + // skip images with center too far from coordinaes if (hypot(Xo,Yo) > 1.5*Radius) continue; // fprintf (stderr, " ** try me **\n"); Index: branches/eam_branches/20090820/Ohana/src/opihi/dvo/imbox.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/dvo/imbox.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/dvo/imbox.c (revision 25766) @@ -3,5 +3,5 @@ int imbox (int argc, char **argv) { - int j, kapa, Nskip, status, InPic, flipped, N, haveNx, haveNy, Nx, Ny, SOLO_PHU; + int j, kapa, Nskip, status, InPic, flipped, N, haveNx, haveNy, Nx, Ny, SOLO_PHU, Npts, NPTS; Vector Xvec, Yvec; double r, d, x[4], y[4], Rmin, Rmax, Rmid; @@ -36,6 +36,8 @@ /* project this image to screen display coords */ - SetVector (&Xvec, OPIHI_FLT, 8); - SetVector (&Yvec, OPIHI_FLT, 8); + Npts = 0; + NPTS = 200; + SetVector (&Xvec, OPIHI_FLT, NPTS); + SetVector (&Yvec, OPIHI_FLT, NPTS); while (gfits_fread_header (f, &header)) { @@ -82,28 +84,30 @@ while (r > Rmid) r-= 360.0; } - status |= RD_to_XY (&Xvec.elements.Flt[2*j], &Yvec.elements.Flt[2*j], r, d, &graphmode.coords); + status |= RD_to_XY (&Xvec.elements.Flt[Npts + 2*j], &Yvec.elements.Flt[Npts + 2*j], r, d, &graphmode.coords); if (j > 0) { - Xvec.elements.Flt[2*j - 1] = Xvec.elements.Flt[2*j]; - Yvec.elements.Flt[2*j - 1] = Yvec.elements.Flt[2*j]; + Xvec.elements.Flt[Npts + 2*j - 1] = Xvec.elements.Flt[Npts + 2*j]; + Yvec.elements.Flt[Npts + 2*j - 1] = Yvec.elements.Flt[Npts + 2*j]; } } - Xvec.elements.Flt[7] = Xvec.elements.Flt[0]; - Yvec.elements.Flt[7] = Yvec.elements.Flt[0]; + Xvec.elements.Flt[Npts + 7] = Xvec.elements.Flt[Npts + 0]; + Yvec.elements.Flt[Npts + 7] = Yvec.elements.Flt[Npts + 0]; + InPic = FALSE; for (j = 0; j < 8; j+=2) { - if ((Xvec.elements.Flt[j] >= graphmode.xmin) && - (Xvec.elements.Flt[j] <= graphmode.xmax) && - (Yvec.elements.Flt[j] >= graphmode.ymin) && - (Yvec.elements.Flt[j] <= graphmode.ymax)) - InPic = TRUE; + if ((Xvec.elements.Flt[Npts + j] >= graphmode.xmin) && + (Xvec.elements.Flt[Npts + j] <= graphmode.xmax) && + (Yvec.elements.Flt[Npts + j] >= graphmode.ymin) && + (Yvec.elements.Flt[Npts + j] <= graphmode.ymax)) + InPic = TRUE; + } + if (!InPic) continue; + + Npts += 8; + if (Npts + 8 >= NPTS) { /* need to leave room for 4 point image */ + NPTS += 200; + REALLOCATE (Xvec.elements.Flt, opihi_flt, NPTS); + REALLOCATE (Yvec.elements.Flt, opihi_flt, NPTS); } - Xvec.Nelements = Yvec.Nelements = 8; - if (InPic) { - graphmode.style = 2; /* points */ - graphmode.ptype = 100; /* connect pairs of points */ - graphmode.etype = 0; - PlotVectorPair (kapa, &Xvec, &Yvec, &graphmode); - } skip: Nskip = gfits_data_size (&header); @@ -111,4 +115,13 @@ gfits_free_header (&header); } + + Xvec.Nelements = Yvec.Nelements = Npts; + if (Npts > 0) { + graphmode.style = 2; /* points */ + graphmode.ptype = 100; /* connect pairs of points */ + graphmode.etype = 0; + PlotVectorPair (kapa, &Xvec, &Yvec, &graphmode); + } + fclose (f); free (Xvec.elements.Flt); Index: branches/eam_branches/20090820/Ohana/src/opihi/dvo/init.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/dvo/init.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/dvo/init.c (revision 25766) @@ -102,5 +102,4 @@ {1, "subpix", subpix, "get subpixel positions"}, {1, "version", version, "show version information"}, -//{1, "imstats", imstats, "plot image statistics"}, //{1, "addxtra", addxtra, "add extra data to object"}, //{1, "getxtra", getxtra, "get extra data from object"}, Index: branches/eam_branches/20090820/Ohana/src/opihi/dvo/photometry.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/dvo/photometry.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/dvo/photometry.c (revision 25766) @@ -325,5 +325,7 @@ return (FALSE); } - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile)) { + + // XXX now that DVO does not allow write access, we can drop the MasterPhotcodeFile + if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, FALSE)) { gprint (GP_ERR, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); return (FALSE); Index: branches/eam_branches/20090820/Ohana/src/opihi/dvo/region_list.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/dvo/region_list.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/dvo/region_list.c (revision 25766) @@ -42,5 +42,5 @@ if (sky != NULL) SkyTableFree (sky); - sky = SkyTableLoadOptimal (CATDIR, skyfile, gscfile, skydepth, verbose); + sky = SkyTableLoadOptimal (CATDIR, skyfile, gscfile, FALSE, skydepth, verbose); if (sky == NULL) return FALSE; Index: branches/eam_branches/20090820/Ohana/src/opihi/dvo/skycoverage.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/dvo/skycoverage.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/dvo/skycoverage.c (revision 25766) @@ -16,4 +16,6 @@ Coords coords; int typehash; + int PhotcodeSelect; + PhotCode *PhotcodeValue; WITH_MOSAIC = FALSE; @@ -43,4 +45,18 @@ remove_argument (N, &argc, argv); ByName = TRUE; + } + + PhotcodeValue = NULL; + PhotcodeSelect = FALSE; + if ((N = get_argument (argc, argv, "-photcode"))) { + if (!InitPhotcodes ()) return (FALSE); + PhotcodeSelect = TRUE; + remove_argument (N, &argc, argv); + PhotcodeValue = GetPhotcodebyName (argv[N]); + if (PhotcodeValue == NULL) { + gprint (GP_ERR, "photcode not found in photcode table\n"); + return (FALSE); + } + remove_argument (N, &argc, argv); } @@ -85,5 +101,6 @@ if (argc != 4) { - gprint (GP_ERR, "USAGE: skycoverage (buffer) (pixscale) (Npts) [-time start range] [-name name]\n"); + gprint (GP_ERR, "USAGE: skycoverage (buffer) (pixscale) (Npts)\n"); + gprint (GP_ERR, " options: [-time start range] [-trange start stop] [-name name] [-photcode name] [+mosaic] [-mosaic] [-ra-center RA]\n"); gprint (GP_ERR, " (buffer) saves bitmapped AIT plot\n"); gprint (GP_ERR, " (pixscale) specifies the pixel size in degrees\n"); @@ -128,6 +145,6 @@ for (xs = 0; xs < Nx; xs++) { status = XY_to_RD (&r, &d, (double)(xs), (double)(ys), &coords); - status &= (r > 0); - status &= (r < 360); + status &= (r >= 0); + status &= (r <= 360); if (status) { V[ys*Nx + xs] = 2; @@ -141,4 +158,13 @@ if (ByName && strcmp (name, image[i].name)) continue; if (TimeSelect && ((image[i].tzero < tzero) || (image[i].tzero+image[i].trate*image[i].NY > tzero + trange))) continue; + + if (PhotcodeSelect) { + if (PhotcodeValue[0].type == PHOT_DEP) { + if (PhotcodeValue[0].code != image[i].photcode) continue; + } else { + if (PhotcodeValue[0].code != GetPhotcodeEquivCodebyCode (image[i].photcode)) continue; + } + } + if (!FindMosaicForImage (image, Nimage, i)) continue; @@ -167,4 +193,6 @@ XY_to_RD (&r, &d, Xi, Yi, &image[i].coords); r = ohana_normalize_angle (r); + if (r - RaCenter > +180.0) r -= 360.0; + if (r - RaCenter < -180.0) r += 360.0; status = RD_to_XY (&Xs, &Ys, r, d, &coords); if (Xs < 0) continue; Index: branches/eam_branches/20090820/Ohana/src/opihi/lib.data/mrqmin.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/lib.data/mrqmin.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/lib.data/mrqmin.c (revision 25766) @@ -17,5 +17,5 @@ int k, j, i; - opihi_flt ydiff, wt, chisq; + opihi_flt ymodel, ydiff, wt, chisq; for (j = 0; j < Npar; j++) { @@ -27,6 +27,9 @@ for (i = 0; i < Npts; i++) { - ydiff = funcs (x[i], par, Npar, dyda) - y[i]; + ymodel = funcs (x[i], par, Npar, dyda); + ydiff = ymodel - y[i]; chisq += SQ(ydiff) * dy[i]; + + // fprintf (stderr, "%f %f - %f : %f -> %f\n", x[i], y[i], ymodel, dy[i], chisq); for (j = 0; j < Npar; j++) { @@ -85,5 +88,5 @@ /* if good, save temp values */ - if (rho > 0) { + if ((chisq > 1e-3) && (rho > -1e-6)) { lambda *= 0.1; ochisq = chisq; Index: branches/eam_branches/20090820/Ohana/src/opihi/lib.shell/command.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/opihi/lib.shell/command.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/opihi/lib.shell/command.c (revision 25766) @@ -5,6 +5,9 @@ int i, status, argc; - char **argv, **targv; + char **argv, **targv, *rawline; Command *cmd; + + rawline = NULL; + // rawline = strcreate (line); /* force a space between ! and first word: !ls becomes ! ls */ @@ -31,5 +34,8 @@ argv = parse_commands (line, &argc); - if (argc == 0) return (TRUE); /* empty command or assignment */ + if (argc == 0) { + FREE (rawline); + return (TRUE); /* empty command or assignment */ + } /* save the original values of argv since command may modify the array */ @@ -54,4 +60,5 @@ msg = get_variable_ptr ("ERRORMSG"); if (msg != (char *) NULL) gprint (GP_ERR, "%s\n", msg); + // gprint (GP_ERR, "error on line: %s\n", rawline); } @@ -62,4 +69,5 @@ # endif + FREE (rawline); return (status); } Index: branches/eam_branches/20090820/Ohana/src/photdbc/src/ConfigInit.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/photdbc/src/ConfigInit.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/photdbc/src/ConfigInit.c (revision 25766) @@ -64,5 +64,5 @@ /* XXX this does not yet write out the master photcode table */ sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile)) { + if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) { fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); exit (1); Index: branches/eam_branches/20090820/Ohana/src/photdbc/src/photdbc.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/photdbc/src/photdbc.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/photdbc/src/photdbc.c (revision 25766) @@ -17,5 +17,5 @@ // the output catalog needs to inherit the SKY_DEPTH of the input catalog - sky = SkyTableLoadOptimal (CATDIR, NULL, GSCFILE, SKY_DEPTH_HST, VERBOSE); + sky = SkyTableLoadOptimal (CATDIR, NULL, GSCFILE, TRUE, SKY_DEPTH_HST, VERBOSE); SkyTableSetFilenames (sky, CATDIR, "cpt"); skylist = SkyListByPatch (sky, -1, ®ION); Index: branches/eam_branches/20090820/Ohana/src/relastro/src/ConfigInit.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/relastro/src/ConfigInit.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/relastro/src/ConfigInit.c (revision 25766) @@ -60,5 +60,5 @@ /* update master photcode table if not defined */ sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile)) { + if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) { fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); exit (1); Index: branches/eam_branches/20090820/Ohana/src/relastro/src/load_images.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/relastro/src/load_images.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/relastro/src/load_images.c (revision 25766) @@ -11,5 +11,5 @@ // load the current sky table (layout of all SkyRegions) - sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, SKY_DEPTH, VERBOSE); + sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); SkyTableSetFilenames (sky, CATDIR, "cpt"); Index: branches/eam_branches/20090820/Ohana/src/relastro/src/relastro_objects.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/relastro/src/relastro_objects.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/relastro/src/relastro_objects.c (revision 25766) @@ -11,5 +11,5 @@ // load the current sky table (layout of all SkyRegions) - sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, SKY_DEPTH, VERBOSE); + sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); SkyTableSetFilenames (sky, CATDIR, "cpt"); Index: branches/eam_branches/20090820/Ohana/src/relphot/src/ConfigInit.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/relphot/src/ConfigInit.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/relphot/src/ConfigInit.c (revision 25766) @@ -58,5 +58,5 @@ /* XXX this does not yet write out the master photcode table */ sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile)) { + if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) { fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); exit (1); Index: branches/eam_branches/20090820/Ohana/src/relphot/src/StarOps.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/relphot/src/StarOps.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/relphot/src/StarOps.c (revision 25766) @@ -193,4 +193,6 @@ for (i = 0; i < Ncatalog; i++) { for (j = 0; j < catalog[i].Naverage; j++) { + + // update average photometry for each of the average filters for (Ns = 0; Ns < Nsecfilt; Ns++) { @@ -207,6 +209,8 @@ if (isnan(Msys)) continue; - // XXX this is a hack for the 2MASS search; better to save an average value? - if (catalog[i].measure[m].psfQual < 0.85) continue; + // XXX only apply this filter for psphot data from GPC1 for now... + if (0 && (catalog[i].measure[m].photcode > 10000) && (catalog[i].measure[m].photcode < 10500)) { + if (catalog[i].measure[m].psfQual < 0.85) continue; + } list[N] = Msys - catalog[i].measure[m].Mcal; @@ -224,4 +228,57 @@ catalog[i].secfilt[Nsecfilt*j+Ns].Ncode = N; catalog[i].secfilt[Nsecfilt*j+Ns].Nused = stats.Nmeas; + } + + // update average flags based on the detection stats. + // XXX we need to clean this up -- this is a serious set of hacks... + if (0) { + int Galaxy2MASS, GalaxySDSS, goodPS1, nEXT, nPSF, good2MASS; + + Galaxy2MASS = FALSE; // best guess for galaxy based on 2MASS J measurements (gal_contam == measure.flags[0x00400000 | 0x00800000]) + GalaxySDSS = FALSE; // best guess for galaxy based on SDSS measurements (XXX need to fix SDSS flags) + goodPS1 = FALSE; // true if any PS1 measurements have psfQual > 0.85 + good2MASS = FALSE; // true if 2MASS J measurements have significant detections + nEXT = nPSF = 0; // number of PS1 PSF vs EXT measurements + + m = catalog[i].average[j].measureOffset; + for (k = 0; k < catalog[i].average[j].Nmeasure; k++, m++) { + + // PS1 data : + if ((catalog[i].measure[m].photcode >= 10000) && (catalog[i].measure[m].photcode <= 10500)) { + if (catalog[i].measure[m].psfQual > 0.85) { + goodPS1 = TRUE; + if (!isnan(catalog[i].measure[m].Map)) { + if (catalog[i].measure[m].M - catalog[i].measure[m].Map > 0.5) { + nEXT ++; + } else { + nPSF ++; + } + } + } + } + + // 2MASS data: + if (catalog[i].measure[m].photcode == 2011) { + if (catalog[i].measure[m].photFlags & 0x00c00000) { + Galaxy2MASS = TRUE; + } + if (catalog[i].measure[m].photFlags & 0x00000007) { + good2MASS = TRUE; + } + } + } + + if (nEXT && (nEXT > nPSF)) { + catalog[i].average[j].flags |= 0x00010000; + } + if (goodPS1) { + catalog[i].average[j].flags |= 0x00020000; + } + if (Galaxy2MASS) { + catalog[i].average[j].flags |= 0x00040000; + } + if (good2MASS) { + catalog[i].average[j].flags |= 0x00080000; + } } } Index: branches/eam_branches/20090820/Ohana/src/relphot/src/load_images.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/relphot/src/load_images.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/relphot/src/load_images.c (revision 25766) @@ -11,5 +11,5 @@ // load the current sky table (layout of all SkyRegions) - sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, SKY_DEPTH, VERBOSE); + sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); SkyTableSetFilenames (sky, CATDIR, "cpt"); Index: branches/eam_branches/20090820/Ohana/src/relphot/src/relphot_objects.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/relphot/src/relphot_objects.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/relphot/src/relphot_objects.c (revision 25766) @@ -11,5 +11,5 @@ // load the current sky table (layout of all SkyRegions) - sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, SKY_DEPTH, VERBOSE); + sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); SkyTableSetFilenames (sky, CATDIR, "cpt"); @@ -68,4 +68,8 @@ if (VERBOSE) fprintf (stderr, "saving catalog %s\n", catalog.filename); + + // we can optionally convert output format here + // but it would be better to define a dvo crawler program to do this + // catalog.catformat = DVO_FORMAT_PS1_V1; dvo_catalog_save (&catalog, VERBOSE); dvo_catalog_unlock (&catalog); Index: branches/eam_branches/20090820/Ohana/src/tools/src/mktemp.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/tools/src/mktemp.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/tools/src/mktemp.c (revision 25766) @@ -1,17 +1,116 @@ # include # include +# include + +# define FALSE 0 +# define TRUE 1 int main (int argc, char **argv) { - if (argc != 2) { - fprintf (stderr, "USAGE: %s (template)\n", argv[0]); - exit (1); + int i, j, Ntotal; + char *tmpdir, *template, deftemplate[32], *prefix, defprefix[32], *filename; + int make_directory, fail_silently, unsafe_mode, full_path; + + tmpdir = NULL; + prefix = NULL; + template = NULL; + filename = NULL; + + make_directory = FALSE; + fail_silently = FALSE; + unsafe_mode = FALSE; + full_path = TRUE; + + for (i = 1; i < argc; i++) { + // -options must be first + if (argv[i][0] == '-') { + if (!strcmp(argv[i], "-V")) { + fprintf (stdout, "mktemp version Ohana $Revision: $\n"); + exit (0); + } + if (!strcmp(argv[i], "-p")) { + if (argc <= i + 1) usage(); + i++; + prefix = argv[i]; + full_path = FALSE; + continue; + } + for (j = 1; j < strlen(argv[i]); j++) { + if (argv[i][j] == 'q') { + fail_silently = TRUE; + continue; + } + if (argv[i][j] == 't') { + full_path = FALSE; + continue; + } + if (argv[i][j] == 'd') { + // make directory + make_directory = TRUE; + continue; + } + if (argv[i][j] == 'u') { + unsafe_mode = TRUE; + continue; + } + // unknown option + usage(); + } + continue; + } + // report an error if too many arguments are given + if (template) usage(); + template = argv[i]; } - if (mkstemp (argv[1]) == -1) exit (1); + if (!full_path) { + // prefix = TMPDIR ? TMPDIR : (prefix ? prefix : /tmp) + tmpdir = getenv("TMPDIR"); + if (tmpdir) { + prefix = tmpdir; + } + if (!prefix) { + strcpy (defprefix, "/tmp"); + prefix = defprefix; + } + if (template && strchr(template, '/')) usage(); + } - fprintf (stdout, "%s\n", argv[1]); + if (!template) { + if (full_path) { + strcpy (deftemplate, "/tmp/tmp.XXXXXXXXXX"); + } else { + strcpy (deftemplate, "tmp.XXXXXXXXXX"); + } + template = deftemplate; + } + + // filename = full_path ? prefix/template : template; + if (!full_path) { + Ntotal = strlen(prefix) + strlen(template) + 2; + filename = (char *) malloc (Ntotal); + snprintf (filename, Ntotal, "%s/%s", prefix, template); + template = filename; + } + + if (make_directory) { + if (mkdtemp (template) == -1) { + if (!fail_silently) fprintf (stderr, "failed to make temp file from %s\n", template); + exit (1); + } + } else { + if (mkstemp (template) == -1) { + if (!fail_silently) fprintf (stderr, "failed to make temp file from %s\n", template); + exit (1); + } + } + + fprintf (stdout, "%s\n", template); exit (0); } +usage() { + fprintf (stderr, "Usage: mktemp [-V] | [-dqtu] [-p prefix] [template]\n"); + exit (1); +} Index: branches/eam_branches/20090820/Ohana/src/uniphot/src/ConfigInit.c =================================================================== --- branches/eam_branches/20090820/Ohana/src/uniphot/src/ConfigInit.c (revision 25206) +++ branches/eam_branches/20090820/Ohana/src/uniphot/src/ConfigInit.c (revision 25766) @@ -36,5 +36,5 @@ /* XXX this does not yet write out the master photcode table */ sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile)) { + if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) { fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); exit (1); Index: branches/eam_branches/20090820/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- branches/eam_branches/20090820/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 25206) +++ branches/eam_branches/20090820/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 25766) @@ -21,5 +21,4 @@ use IPC::Cmd 0.36 qw( can_run run ); - use PS::IPP::Metadata::List qw( parse_md_list ); use PS::IPP::Config qw( :standard ); @@ -35,6 +34,8 @@ my $img_type = shift; # required my $id = shift; # required unless req_type eq bycoord or byskycell + my $tess_id = shift; my $component= shift; # class_id or skycell_id my $inverse = shift; + my $need_magic = shift; my $x = shift; my $y = shift; @@ -42,10 +43,24 @@ my $mjd_max = shift; my $filter = shift; + my $label = shift; my $verbose = shift; # we die in response to bad data in request files - die "Unknown req_type: $req_type" if ($req_type ne "byid") and ($req_type ne "byexp") - and ($req_type ne "bycoord") and ($req_type ne "bydiff"); + die "Unknown req_type: $req_type" + if ($req_type ne "byid") and + ($req_type ne "byexp") and + ($req_type ne "bycoord") and + ($req_type ne "bydiff") and + ($req_type ne "byskycell"); + + my $dateobs_begin; + my $dateobs_end; + if (!iszero($mjd_min)) { + $dateobs_begin = mjd_to_dateobs($mjd_min); + } + if (!iszero($mjd_max)) { + $dateobs_end = mjd_to_dateobs($mjd_max); + } if (($req_type eq "byid") and ($img_type eq "diff")) { @@ -77,4 +92,5 @@ $req_type = "byid"; $id = $image->{warp_id}; + $component = $image->{skycell_id}; return undef if !$id; # fall through and lookup by warp_id @@ -95,5 +111,5 @@ # regtool -dbname $image_db -processedimfile -time_begin $mjd_min -time_end = $mjd_max -filter $filter # - my $results = lookup_bycoord($ipprc, $image_db, $x, $y, $mjd_min, $mjd_max, $filter, $verbose); + my $results = lookup_bycoord($ipprc, $image_db, $x, $y, $dateobs_begin, $dateobs_end, $filter, $verbose); # now take the results and lookup byexp @@ -102,7 +118,16 @@ $req_type = "byexp"; $id = $results->{exp_name}; - } - - my $results = lookup($ipprc, $image_db, $req_type, $img_type, $id, $component, $verbose); + } elsif ($req_type eq "byskycell") { + if (($img_type eq "raw") or ($img_type eq "chip")) { + print STDERR "REQ_TYPE byskycell not supported for IMG_TYPE raw or chip\n"; + return undef; + } + if (!$tess_id or !$component) { + print STDERR "component and tess_id are required for REQ_TYPE byskycell\n"; + return undef; + } + } + + my $results = lookup($ipprc, $image_db, $req_type, $img_type, $id, $tess_id, $component, $need_magic, $dateobs_begin, $dateobs_end, $filter, $label, $verbose); return $results; @@ -115,5 +140,11 @@ my $img_type = shift; my $id = shift; + my $tess_id = shift; my $component= shift; + my $need_magic = shift; + my $dateobs_begin = shift; + my $dateobs_end = shift; + my $filter = shift; + my $label = shift; my $verbose = shift; @@ -139,9 +170,9 @@ my $skycell_id; - # special class_id value "null" means ignore - if ($component and ($component eq "null")) { + if (isnull($component)) { $component = undef; } + my $magic_arg = $need_magic ? " -destreaked" : ""; if ($img_type eq "raw") { $class_id = $component; @@ -196,7 +227,18 @@ } elsif ($req_type eq "byexp") { $command .= " -exp_name $id"; + } elsif ($req_type eq "byskycell") { + $command .= " -tess_id $tess_id -skycell_id $skycell_id"; } else { die "Unknown req_type supplied: $req_type"; } + + if ($img_type ne "stack") { + $command .= $magic_arg; + $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin; + $command .= " -dateobs_end $dateobs_end" if $dateobs_end; + } + + $command .= " -filter $filter" if !isnull($filter); + $command .= " -label $label" if !isnull($label); # run the tool and parse the output @@ -233,4 +275,9 @@ # just above probably solved the problem I was trying to solve with this test) next if !$base; + } else { + # raw images don't have path_base yet. Manufacture one + my $uri = $image->{uri}; + my $path_base = $uri =~ s/\.fits//g; + $image->{$path_base} = $path_base; } if (!$camera) { @@ -381,6 +428,6 @@ my $x = shift; my $y = shift; - my $mjd_min = shift; - my $mjd_max = shift; + my $dateobs_begin = shift; + my $dateobs_end = shift; my $filter = shift; my $verbose = shift; @@ -398,13 +445,11 @@ my $args; - if ($mjd_min) { - my $dateobs_min = mjd_to_dateobs($mjd_min); - $args .= " -dateobs_begin $dateobs_min"; - } - if ($mjd_max) { - my $dateobs_max = mjd_to_dateobs($mjd_max); - $args .= " -dateobs_end $dateobs_max"; - } - if ($filter) { + if (!isnull($dateobs_begin)) { + $args .= " -dateobs_begin $dateobs_begin"; + } + if (!isnull($dateobs_end)) { + $args .= " -dateobs_end $dateobs_end"; + } + if (!isnull($filter)) { $args .= " -filter $filter"; } @@ -562,5 +607,16 @@ my ($sec, $min, $hr, $day, $mon, $year) = gmtime($ticks); - return sprintf "'%4d-%02d-%02d %02d:%02d:%02d'", $year+1900, $mon+1, $day, $hr, $min, $sec; + return sprintf "'%4d-%02d-%02dT%02d:%02d:%02dZ'", $year+1900, $mon+1, $day, $hr, $min, $sec; +} + +sub isnull { + my $val = shift; + + return (!defined($val) or (lc($val) eq "null")); +} + +sub iszero { + my $val = shift; + return (!defined($val) or ($val == 0)); } Index: branches/eam_branches/20090820/dbconfig/add.md =================================================================== --- branches/eam_branches/20090820/dbconfig/add.md (revision 25766) +++ branches/eam_branches/20090820/dbconfig/add.md (revision 25766) @@ -0,0 +1,25 @@ +addRun METADATA + add_id S64 0 # Primary Key AUTO_INCREMENT + cam_id S64 0 # Key INDEX(add_id,cam_id) fkey(cam_id) ref camRun(cam_id) + state STR 64 # key + workdir STR 255 + workdir_state STR 64 + label STR 64 + dvodb STR 255 + magicked S64 0 +END + +addProcessedExp METADATA + add_id S64 0 # Primary Key AUTO_INCREMENT + dtime_addstar F32 0.0 + + n_stars S32 0 + + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +addMask METADATA + label STR 64 # Primary Key +END + Index: branches/eam_branches/20090820/dbconfig/changes.txt =================================================================== --- branches/eam_branches/20090820/dbconfig/changes.txt (revision 25206) +++ branches/eam_branches/20090820/dbconfig/changes.txt (revision 25766) @@ -1082,5 +1082,5 @@ ALTER TABLE rcRun ADD COLUMN fault SMALLINT DEFAULT 0; -ALTER TABLE rcRun ADD COLUMN status_fs VARCHAR(64) AFTER state; +ALTER TABLE rcRun ADD COLUMN status_fs_name VARCHAR(64) AFTER state; ALTER TABLE diffSkyfile ADD COLUMN deconv_max FLOAT AFTER kernel_yy; @@ -1206,2 +1206,96 @@ ALTER TABLE magicDSRun ADD COLUMN inv_magic_id BIGINT AFTER magic_id; ALTER TABLE magicDSRun ADD FOREIGN KEY (inv_magic_id) REFERENCES magicRun(magic_id); + +-- Adding columns for debugging publishing problems + +ALTER TABLE publishDone ADD COLUMN hostname VARCHAR(64) AFTER path_base; +ALTER TABLE publishDone ADD COLUMN dtime_script FLOAT AFTER hostname; + +-- Version 1.1.55 +-- addstar stage tables: + +CREATE TABLE addRun ( + add_id BIGINT AUTO_INCREMENT, + cam_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + dvodb VARCHAR(255), + magicked BIGINT, + PRIMARY KEY(add_id), + KEY(add_id), + KEY(cam_id), + KEY(state), + KEY(workdir_state), + KEY(label), + INDEX(add_id, cam_id), + FOREIGN KEY(cam_id) REFERENCES camRun(cam_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE addProcessedExp ( + add_id BIGINT AUTO_INCREMENT, + dtime_addstar FLOAT, + n_stars INT, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(add_id), + FOREIGN KEY(add_id) REFERENCES addRun(add_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE addMask ( + label VARCHAR(64), + PRIMARY KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + +ALTER TABLE pzDownloadImfile ADD COLUMN bytes INT; +ALTER TABLE pzDownloadImfile ADD COLUMN md5sum VARCHAR(32); +ALTER TABLE newImfile ADD COLUMN bytes INT; +ALTER TABLE newImfile ADD COLUMN md5sum VARCHAR(32); +ALTER TABLE rawImfile ADD COLUMN bytes INT; +ALTER TABLE rawImfile ADD COLUMN md5sum VARCHAR(32); +ALTER TABLE rawImfile ADD COLUMN burntool_state SMALLINT; + +-- Vesion 1.1.56 + +-- Merge the rcDSProduct and rcDestination tables + +ALTER TABLE rcDestination ADD COLUMN dbname varchar(64) AFTER last_fileset; +ALTER TABLE rcDestination ADD COLUMN dbhost varchar(64) AFTER dbname; + +UPDATE rcDestination JOIN rcDSProduct USING(prod_id) + SET rcDestination.name = rcDSProduct.name, + rcDestination.dbname = rcDSProduct.dbname, + rcDestination.dbhost = rcDSProduct.dbhost; + +ALTER TABLE rcDestination DROP foreign key rcDestination_ibfk_1 ; +ALTER TABLE rcDestination DROP key prod_id; +ALTER TABLE rcDestination DROP COLUMN prod_id; + +-- change references to rcDSProduct in rcDSFileset to reference rcDestination + +ALTER TABLE rcDSFileset DROP foreign key rcDSFileset_ibfk_2; +ALTER TABLE rcDSFileset DROP key prod_id; +ALTER TABLE rcDSFileset CHANGE COLUMN prod_id dest_id BIGINT NOT NULL DEFAULT '0'; +ALTER TABLE rcDSFileset ADD KEY (dest_id); +ALTER TABLE rcDSFileset ADD FOREIGN KEY (dest_id) REFERENCES rcDestination(dest_id); + +-- finally drop the obsolete table +DROP TABLE rcDSProduct; + +-- The following changes should have been made to the gpc1 database ages ago +-- do not apply if these columns exist +ALTER TABLE pstampRequest DROP COLUMN outFileset; +ALTER TABLE pstampRequest ADD COLUMN name VARCHAR(64) UNIQUE AFTER state; +ALTER TABLE pstampRequest ADD COLUMN reqType VARCHAR(16) AFTER name; +ALTER TABLE pstampRequest ADD COLUMN outProduct VARCHAR(64) after reqType; +ALTER TABLE pstampRequest ADD COLUMN fault SMALLINT AFTER uri; +-- end of old changes + +-- This change is new +ALTER TABLE pstampRequest ADD COLUMN label VARCHAR(64) AFTER reqType; + +ALTER TABLE distRun ADD COLUMN magic_ds_id BIGINT AFTER stage_id; + +ALTER TABLE pstampJob ADD COLUMN options BIGINT; Index: branches/eam_branches/20090820/dbconfig/config.md =================================================================== --- branches/eam_branches/20090820/dbconfig/config.md (revision 25206) +++ branches/eam_branches/20090820/dbconfig/config.md (revision 25766) @@ -2,4 +2,4 @@ pkg_name STR ippdb pkg_namespace STR ippdb - pkg_version STR 1.1.54 + pkg_version STR 1.1.55 END Index: branches/eam_branches/20090820/dbconfig/dist.md =================================================================== --- branches/eam_branches/20090820/dbconfig/dist.md (revision 25206) +++ branches/eam_branches/20090820/dbconfig/dist.md (revision 25766) @@ -14,4 +14,5 @@ stage STR 64 stage_id S64 0 + magic_ds_id S64 0 label STR 64 # Key outroot STR 255 Index: branches/eam_branches/20090820/dbconfig/ipp.m4 =================================================================== --- branches/eam_branches/20090820/dbconfig/ipp.m4 (revision 25206) +++ branches/eam_branches/20090820/dbconfig/ipp.m4 (revision 25766) @@ -14,4 +14,5 @@ include(chip.md) include(cam.md) +include(add.md) include(fake.md) include(warp.md) Index: branches/eam_branches/20090820/dbconfig/new.md =================================================================== --- branches/eam_branches/20090820/dbconfig/new.md (revision 25206) +++ branches/eam_branches/20090820/dbconfig/new.md (revision 25766) @@ -22,4 +22,6 @@ uri STR 255 epoch UTC 0001-01-01T00:00:00Z + bytes S32 0 + md5sum STR 32 END Index: branches/eam_branches/20090820/dbconfig/pstamp.md =================================================================== --- branches/eam_branches/20090820/dbconfig/pstamp.md (revision 25206) +++ branches/eam_branches/20090820/dbconfig/pstamp.md (revision 25766) @@ -24,4 +24,5 @@ name STR 64 # UNIQUE reqType STR 16 + label STR 64 outProduct STR 64 uri STR 255 @@ -38,3 +39,4 @@ exp_id S64 0 outputBase STR 255 + options S64 64 END Index: branches/eam_branches/20090820/dbconfig/publish.md =================================================================== --- branches/eam_branches/20090820/dbconfig/publish.md (revision 25206) +++ branches/eam_branches/20090820/dbconfig/publish.md (revision 25766) @@ -1,23 +1,25 @@ # Tables for publishing data to a science client -publishClient METADATA - client_id S64 0 # Primary Key AUTO_INCREMENT - product STR 64 - stage STR 64 - workdir STR 255 - comment STR 255 +publishClient METADATA + client_id S64 0 # Primary Key AUTO_INCREMENT + product STR 64 + stage STR 64 + workdir STR 255 + comment STR 255 +END + +publishRun METADATA + pub_id S64 0 # Primary Key AUTO_INCREMENT + client_id S64 0 + stage_id S64 0 + label STR 64 + state STR 64 +END + +publishDone METADATA + pub_id S64 0 # Primary Key + path_base STR 255 + hostname STR 64 + dtime_script F32 0.0 + fault S16 0 END - -publishRun METADATA - pub_id S64 0 # Primary Key AUTO_INCREMENT - client_id S64 0 - stage_id S64 0 - label STR 64 - state STR 64 -END - -publishDone METADATA - pub_id S64 0 # Primary Key - path_base STR 255 - fault S16 0 -END Index: branches/eam_branches/20090820/dbconfig/raw.md =================================================================== --- branches/eam_branches/20090820/dbconfig/raw.md (revision 25206) +++ branches/eam_branches/20090820/dbconfig/raw.md (revision 25766) @@ -136,3 +136,6 @@ epoch UTC 0001-01-01T00:00:00Z magicked S64 0 + bytes S32 0 + md5sum STR 32 + burntool_state S16 0 END Index: branches/eam_branches/20090820/dbconfig/rc.md =================================================================== --- branches/eam_branches/20090820/dbconfig/rc.md (revision 25206) +++ branches/eam_branches/20090820/dbconfig/rc.md (revision 25766) @@ -1,7 +1,11 @@ -rcDSProduct METADATA - prod_id S64 0 # Primary Key - name STR 64 - dbname STR 64 - dbhost STR 64 +rcDestination METADATA + dest_id S64 0 # Primary Key + name STR 64 + status_uri STR 255 + comment STR 255 + last_fileset STR 255 + dbname STR 64 + dbhost STR 64 + state STR 64 # Key END @@ -9,5 +13,5 @@ fs_id S64 0 # Primary Key dist_id S64 0 # fkey(dist_id) ref distRun(dist_id) - prod_id S64 0 # fkey(prod_id) ref rcDSProduct(prod_id) + dest_id S64 0 # fkey(dest_id) ref rcDestination(dest_id) name STR 255 state STR 64 @@ -15,13 +19,4 @@ END -rcDestination METADATA - dest_id S64 0 # Primary Key - prod_id S64 0 # fkey(prod_id) ref rcDSProduct(prod_id) - name STR 64 - status_uri STR 255 - comment STR 255 - last_fileset STR 255 - state STR 64 # Key -END rcInterest METADATA Index: branches/eam_branches/20090820/dbconfig/summitcopy.md =================================================================== --- branches/eam_branches/20090820/dbconfig/summitcopy.md (revision 25206) +++ branches/eam_branches/20090820/dbconfig/summitcopy.md (revision 25766) @@ -57,4 +57,6 @@ epoch UTC 0001-01-01T00:00:00Z hostname STR 64 + bytes S32 0 + md5sum STR 32 END Index: branches/eam_branches/20090820/extsrc/gpcsw/Make.Common.fixed =================================================================== --- branches/eam_branches/20090820/extsrc/gpcsw/Make.Common.fixed (revision 25206) +++ branches/eam_branches/20090820/extsrc/gpcsw/Make.Common.fixed (revision 25766) @@ -670,6 +670,6 @@ $(DIR_BIN)/%: $(DIR_BIN)/%-$(VERSION) @echo "--> Installing link to $(notdir $<) as $(notdir $@)" - @-mv $@ $@.$(shell date +%H%M%S).TEXT_FILE_BUSY 2> /dev/null || /bin/true - @-rm -f $(DIR_BIN)/*.TEXT_FILE_BUSY 2> /dev/null || /bin/true + @-mv $@ $@.$(shell date +%H%M%S).TEXT_FILE_BUSY 2> /dev/null || true + @-rm -f $(DIR_BIN)/*.TEXT_FILE_BUSY 2> /dev/null || true @ln $< $@ # @-setuidinst $@ @@ -677,10 +677,10 @@ $(DIR_BIN)/%-$(VERSION): scripts/%.sh @echo "--> Installing sh script $(notdir $@)" - @-mv $@ $@.$(shell date +%H%M%S).TEXT_FILE_BUSY 2> /dev/null || /bin/true + @-mv $@ $@.$(shell date +%H%M%S).TEXT_FILE_BUSY 2> /dev/null || true @$(INSTALL) -m 0555 $< $@ || ( rm -f $@ && exit 1 ) $(DIR_BIN)/%-$(VERSION): $(EXECNAME) @echo "--> Installing program $(notdir $@)" - @-mv $@ $@.$(shell date +%H%M%S).TEXT_FILE_BUSY 2> /dev/null || /bin/true + @-mv $@ $@.$(shell date +%H%M%S).TEXT_FILE_BUSY 2> /dev/null || true @$(INSTSTRIP) -m 0755 $< $@ || ( rm -f $@ && exit 1 ) @chmod 0555 $@ Index: branches/eam_branches/20090820/extsrc/gpcsw/gpcsrc/fits/burntool/burnparams.h =================================================================== --- branches/eam_branches/20090820/extsrc/gpcsw/gpcsrc/fits/burntool/burnparams.h (revision 25206) +++ branches/eam_branches/20090820/extsrc/gpcsw/gpcsrc/fits/burntool/burnparams.h (revision 25766) @@ -55,3 +55,5 @@ EXTERN int EXPIRE_TRAIL_TIME; /* Expire trails after this interval */ +EXTERN int PERSIST_RETAIN; /* Retain bad-slope persistence fits */ + #endif /* _INCLUDED_burnparams_ */ Index: branches/eam_branches/20090820/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c =================================================================== --- branches/eam_branches/20090820/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c (revision 25206) +++ branches/eam_branches/20090820/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c (revision 25766) @@ -100,4 +100,6 @@ EXPIRE_TRAIL_TIME = 2000; /* Expire a persist after this [sec] */ + PERSIST_RETAIN = 0; /* Retain persists with bad slopes? */ + /* Parse the args */ cellxy = -1; @@ -175,4 +177,8 @@ persistfitsfile = argv[i] + 7; +/* Keep persistence streaks which had a bad slope? */ + } else if(strncmp(argv[i], "persist=", 8) == 0) {/* persist={t|f} */ + PERSIST_RETAIN = argv[i][8] == 'y' || argv[i][8] == '1' || argv[i][8] == 't'; + /* Output file for PSF gallery */ } else if(strncmp(argv[i], "psf=", 4) == 0) { /* psf=fname */ @@ -286,4 +292,18 @@ CONCAT_FITS = argv[i][10] == 'n' || argv[i][10] == '0' || argv[i][10] == 'f'; + +/* Set the e/ADU for the noise gate that cells must pass */ + } else if(strncmp(argv[i], "EADU=", 5) == 0) { /* EADU=e */ + if(sscanf(argv[i]+5, "%lf", &MIN_EADU) != 1) { + fprintf(stderr, "\rerror: cannot get e/ADU size from `%s'\n", argv[i]); + exit(EXIT_FAILURE); + } + +/* Set the read noise for the noise gate that cells must pass */ + } else if(strncmp(argv[i], "RN=", 3) == 0) { /* EADU=e */ + if(sscanf(argv[i]+3, "%d", &MAX_READ_NOISE) != 1) { + fprintf(stderr, "\rerror: cannot get read noise size from `%s'\n", argv[i]); + exit(EXIT_FAILURE); + } /* Quiet? */ @@ -580,10 +600,11 @@ /* Fix up the streaks (don't bother if table only) */ - if(!tableonly) { +// Why the heck do I get to skip this if table only? Makes no sense! +// if(!tableonly) { persist_fix(naxis1-ovrscan1, naxis2-ovrscan2, naxis1, buf, OTA+cell); /* Tell us about it? */ if(VERBOSE & VERB_NORM) persist_blab(OTA+cell); - } +// } } Index: branches/eam_branches/20090820/extsrc/gpcsw/gpcsrc/fits/burntool/man/burntool.1 =================================================================== --- branches/eam_branches/20090820/extsrc/gpcsw/gpcsrc/fits/burntool/man/burntool.1 (revision 25206) +++ branches/eam_branches/20090820/extsrc/gpcsw/gpcsrc/fits/burntool/man/burntool.1 (revision 25766) @@ -53,4 +53,12 @@ power laws; downward burns as exponentials. + The fit to downward, persistence burns may have an unreasonable "slope" + (meaning exponential sign), increasing away from the burn origin. + These are discarded unless "persist=t" is invoked. It is possible + for the fit to be fooled by uncataloged stars, so the persistence + can be kept in the output file with "persist=t" (although no fit + correction is applied) and they will propagate from input to output + until they finally achieve a legal fit with negligible amplitude. + Burntool also identifies really blasted areas which are saturated from top to bottom. These cannot be fitted, but are carried along for @@ -231,4 +239,8 @@ 'in' takes precedence. + persist={t|f} + Retain persistence streaks whose fit slope turned out to be + unreasonable. + out=fname Output file for burn streaks @@ -290,4 +302,15 @@ expire=N Retire a blasted burn after N seconds + + EADU=x + Set the minimum e/ADU for acceptable cell sky noise + (default 0.3). Should not normally be changed! + RN=x + Set the maximum read noise for acceptable cell sky noise + (default 20). Should not normally be changed! + + Any cell whose noise variance in the sky is greater than + sky/EADU+RN*RN is rejected out of hand, and skipped for + further processing. quiet={t|f} @@ -332,4 +355,5 @@ BUGS: 090224: Still in development + 090810: Squished a few memory bugs SEE ALSO: Index: branches/eam_branches/20090820/extsrc/gpcsw/gpcsrc/fits/burntool/persist_fits.c =================================================================== --- branches/eam_branches/20090820/extsrc/gpcsw/gpcsrc/fits/burntool/persist_fits.c (revision 25206) +++ branches/eam_branches/20090820/extsrc/gpcsw/gpcsrc/fits/burntool/persist_fits.c (revision 25766) @@ -222,6 +222,14 @@ for(k=0; k 0) { + boxbuf[nbox].zero = (double *)calloc(boxbuf[nbox].nfit, sizeof(double)); + boxbuf[nbox].xfit = (int *)calloc(boxbuf[nbox].nfit, sizeof(int)); + boxbuf[nbox].yfit = (int *)calloc(boxbuf[nbox].nfit, sizeof(int)); + for(i=0; i 0) zk = box[k].zero[box[k].nfit/2]; for(jp=kp+1; jp 0) zj = box[j].zero[box[j].nfit/2]; if(ABS(yctr[jp]-yctr[kp]) > DIFFERENT_STREAK) { /* Trim back the feebler streak */ @@ -238,6 +241,15 @@ /* First: patched up persists */ for(k=0; k= FIT_MAX_SLOPE || slope < FIT_MIN_SLOPE) { + box->slope = slope; + box->nfit = 0; box->fiterr = FIT_SLOPE_ERROR; return(-1); Index: branches/eam_branches/20090820/extsrc/gpcsw/gpcsrc/fits/libfh/fh.c =================================================================== --- branches/eam_branches/20090820/extsrc/gpcsw/gpcsrc/fits/libfh/fh.c (revision 25206) +++ branches/eam_branches/20090820/extsrc/gpcsw/gpcsrc/fits/libfh/fh.c (revision 25766) @@ -1039,4 +1039,5 @@ newval[strlen(newval) - 1] = '\0'; fh_set_str(hu, idx, name, newval, comment); + free(newval); return; } @@ -1519,4 +1520,29 @@ i++; } + return FH_SUCCESS; +} + +static void +pad_header(HeaderUnit hu, int n) +{ + int i; + double idx = 900000000; + + for (i = 0; i < n; i++) + { + fh_set_card(hu, idx++, FH_RESERVE); + } +} + +fh_result +fh_copy(HeaderUnit hu, const HeaderUnit source_) +{ + fh_result result; + int reserve; + + result = fh_merge(hu, source_); + if (result != FH_SUCCESS) return result; + reserve = fh_get_reserve(source_); + if (reserve) pad_header(hu, reserve); return FH_SUCCESS; } @@ -1757,4 +1783,21 @@ } +int +fh_get_reserve(HeaderUnit hu) +{ + HeaderUnitStruct* list = FH_HU(hu); + + if (!list) + { + log_error("invalid argument to fh_get_reserve()"); + return -1; + } + + if (list->reserve) + return list->reserve; + else + return list->reserve_found; +} + fh_result fh_rewrite(HeaderUnit hu) Index: branches/eam_branches/20090820/extsrc/gpcsw/gpcsrc/fits/libfh/fh.h =================================================================== --- branches/eam_branches/20090820/extsrc/gpcsw/gpcsrc/fits/libfh/fh.h (revision 25206) +++ branches/eam_branches/20090820/extsrc/gpcsw/gpcsrc/fits/libfh/fh.h (revision 25766) @@ -174,4 +174,9 @@ * is used to create a new FITS header. NOT for use with fh_rewrite(). */ +int fh_get_reserve(HeaderUnit hu); +/* + * Get the current setting or the number of reserve cards found in + * a header unit. Returns -1 if hu is invalid. + */ fh_result fh_validate(HeaderUnit hu); /* fh_validate.c; for use by fhtool.c */ @@ -386,4 +391,5 @@ double fh_idx(HeaderUnit hu); /* idx of the last card returned by fh_next */ fh_result fh_merge(HeaderUnit hu, const HeaderUnit source); /* source unchanged */ +fh_result fh_copy(HeaderUnit hu, const HeaderUnit source); /* fh_merge+keep reserve */ /* --------------------------------------------------------- @@ -459,4 +465,3 @@ */ - #endif /* _INCLUDED_fh */ Index: branches/eam_branches/20090820/ippMonitor/raw/Login.php =================================================================== --- branches/eam_branches/20090820/ippMonitor/raw/Login.php (revision 25206) +++ branches/eam_branches/20090820/ippMonitor/raw/Login.php (revision 25766) @@ -44,5 +44,5 @@ echo "You are now logged out
\n"; logintext (); - loginform (); + loginform (""); menu_end (); exit (); Index: branches/eam_branches/20090820/ippScripts/Build.PL =================================================================== --- branches/eam_branches/20090820/ippScripts/Build.PL (revision 25206) +++ branches/eam_branches/20090820/ippScripts/Build.PL (revision 25766) @@ -49,4 +49,5 @@ scripts/chip_imfile.pl scripts/camera_exp.pl + scripts/addstar_run.pl scripts/fake_imfile.pl scripts/warp_overlap.pl Index: branches/eam_branches/20090820/ippScripts/scripts/addstar_run.pl =================================================================== --- branches/eam_branches/20090820/ippScripts/scripts/addstar_run.pl (revision 25766) +++ branches/eam_branches/20090820/ippScripts/scripts/addstar_run.pl (revision 25766) @@ -0,0 +1,250 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +#my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); +my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +#my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +#my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +#my $psastro = can_run('psastro') or (warn "Can't find psastro" and $missing_tools = 1); +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $recipe, $dbname, $reduction, $dvodb, $verbose, $no_update, + $no_op, $redirect, $save_temps, $run_state); +GetOptions( + 'exp_tag=s' => \$exp_tag, # Exposure identifier + 'add_id=s' => \$add_id, # Camtool identifier + 'recipe=s' => \$recipe, # Recipe to use + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'outroot|w=s' => \$outroot, # output file base name + 'camroot|w=s' => \$camroot, # camera stage root name. + 'reduction=s' => \$reduction, # Reduction class + 'dvodb|w=s' => \$dvodb, # output DVO database + 'run-state=s' => \$run_state, # 'new' or 'update' + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --exp_tag --add_id --camera --outroot", + -exitval => 3, + ) unless + defined $exp_tag and + defined $add_id and + defined $outroot and + defined $camera; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $add_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + +if (not defined $run_state) { $run_state = 'new'; } +if ($run_state eq 'update') { + $logDest .= '.update'; +} + +if ($redirect) { + $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $add_id, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; + +my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use +&my_die("Unrecognised ADDSTAR recipe", $add_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar; + +#my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use +#&my_die("Unrecognised PSASTRO recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +my $cmdflags; + +# Get list of component files +my $files; # Array of component files +{ + my $command = "$addtool -pendingexp -add_id $add_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addtool: $error_code", $add_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $add_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); +} + +my $chipObjectsExist = 0; + +# Output products +$ipprc->outroot_prepare($outroot); + +# the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs +my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $camroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + +if ($run_state eq 'update') { + $traceDest .= '.update'; +} + +# convert supplied DVO database name to UNIX filename +my $dvodbReal; +if (defined $dvodb) { + $dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO + $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ); +} + +my $dtime_addstar = 0; + +unless ($no_op) { + if (defined $dvodbReal and ($run_state eq 'new')) { + ## XXX the camera analysis can either save the full set of + ## detections, or just the image metadata, in the dvodb + + ## get the addstar recipe for this camera and CAMERA reduction + my $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $add_id, $PS_EXIT_SYS_ERROR); + } + my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $add_id, $PS_EXIT_SYS_ERROR); + + ## allow the dvodb to save only images, or the full detection set + my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY'); + + # XXX this construct requires the user to have a valid .ptolemyrc + # XXX which in turn points at ippconfig/dvo.site + # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) + # XXX this needs to be converted to addstar_client... + + my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar + $command = "$addstar -D CAMERA $camdir -update"; + $command .= " -image" if $imagesOnly; + $command .= " -D CATDIR $dvodbReal"; + + my $realFile = $ipprc->file_resolve($fpaObjects); + $command .= " $realFile"; + + my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $add_id, $error_code); + } + $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script + } +} + + +# This needs to be updated when addtool is written. BROKEN +my $fpaCommand = "$addtool -add_id $add_id"; +if ($run_state eq 'new') { + $fpaCommand .= " -addprocessedexp"; + $fpaCommand .= " -path_base $outroot"; + $fpaCommand .= " $cmdflags"; + $fpaCommand .= " -dtime_addstar $dtime_addstar"; +} else { + $fpaCommand .= " -updaterun -state full"; +} +$fpaCommand .= " -dbname $dbname" if defined $dbname; + +# Add the result into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to add result to database: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $fpaCommand\n"; +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $add_id = shift; # Camtool identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + if (defined $add_id and not $no_update) { +# This needs to be updated when addtool is written. BROKEN + my $command = "$addtool -add_id $add_id"; + if ($run_state eq 'new') { + $command .= " -addprocessedexp"; + $command .= " -uri UNKNOWN"; + $command .= " -fault $exit_code"; + $command .= " -path_base $outroot"; + $command .= " -path_base $outroot" if defined $outroot; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + } else { + $command .= " -updateprocessedexp"; + $command .= " -fault $exit_code"; + } + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: branches/eam_branches/20090820/ippScripts/scripts/camera_exp.pl =================================================================== --- branches/eam_branches/20090820/ippScripts/scripts/camera_exp.pl (revision 25206) +++ branches/eam_branches/20090820/ippScripts/scripts/camera_exp.pl (revision 25766) @@ -93,6 +93,6 @@ &my_die("Unrecognised JPEG recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe2; -my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use -&my_die("Unrecognised ADDSTAR recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar; +#my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use +#&my_die("Unrecognised ADDSTAR recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar; my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use @@ -205,5 +205,5 @@ } -my $dtime_addstar = 0; +#my $dtime_addstar = 0; unless ($no_op) { @@ -297,47 +297,47 @@ # run addstar on the output fpaObjects (if a DVO database is defined) - if (defined $dvodbReal and ($run_state eq 'new')) { - - ## XXX the camera analysis can either save the full set of - ## detections, or just the image metadata, in the dvodb - - ## get the addstar recipe for this camera and CAMERA reduction - $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_SYS_ERROR); - } - my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_SYS_ERROR); - - ## allow the dvodb to save only images, or the full detection set - my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY'); - - # XXX this construct requires the user to have a valid .ptolemyrc - # XXX which in turn points at ippconfig/dvo.site - # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) - # XXX this needs to be converted to addstar_client... - - my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar - my $command; - $command = "$addstar -D CAMERA $camdir -update"; - $command .= " -image" if $imagesOnly; - $command .= " -D CATDIR $dvodbReal"; - - my $realFile = $ipprc->file_resolve($fpaObjects); - $command .= " $realFile"; - - my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform addstar: $error_code", $cam_id, $error_code); - } - $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script - } +# if (defined $dvodbReal and ($run_state eq 'new')) { + +# ## XXX the camera analysis can either save the full set of +# ## detections, or just the image metadata, in the dvodb + +# ## get the addstar recipe for this camera and CAMERA reduction +# $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -"; +# ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run(command => $command, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_SYS_ERROR); +# } +# my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or +# &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_SYS_ERROR); + +# ## allow the dvodb to save only images, or the full detection set +# my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY'); + +# # XXX this construct requires the user to have a valid .ptolemyrc +# # XXX which in turn points at ippconfig/dvo.site +# # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) +# # XXX this needs to be converted to addstar_client... + +# my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar +# my $command; +# $command = "$addstar -D CAMERA $camdir -update"; +# $command .= " -image" if $imagesOnly; +# $command .= " -D CATDIR $dvodbReal"; + +# my $realFile = $ipprc->file_resolve($fpaObjects); +# $command .= " $realFile"; + +# my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run(command => $command, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform addstar: $error_code", $cam_id, $error_code); +# } +# $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script +# } } } @@ -353,5 +353,5 @@ $fpaCommand .= " -hostname $host" if defined $host; $fpaCommand .= " -dtime_script $dtime_script"; - $fpaCommand .= " -dtime_addstar $dtime_addstar"; +# $fpaCommand .= " -dtime_addstar $dtime_addstar"; } else { $fpaCommand .= " -updaterun -state full"; Index: branches/eam_branches/20090820/ippScripts/scripts/chip_imfile.pl =================================================================== --- branches/eam_branches/20090820/ippScripts/scripts/chip_imfile.pl (revision 25206) +++ branches/eam_branches/20090820/ippScripts/scripts/chip_imfile.pl (revision 25766) @@ -19,4 +19,5 @@ use IPC::Cmd 0.36 qw( can_run run ); use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); use PS::IPP::Config 1.01 qw( :standard ); use File::Temp qw( tempfile ); @@ -143,7 +144,142 @@ ## XXX make the feature more general? my $useDeburnedImage = metadataLookupBool($recipeData, 'USE.DEBURNED.IMAGE'); - if ($useDeburnedImage && $deburned) { - $uri =~ s/fits$/burn.fits/; - &my_die("Couldn't find deburned input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri); + + if ($useDeburnedImage) { + my $useBestBurntool = metadataLookupBool($recipeData, 'USE.BEST.BURNTOOL'); + + ## Check that we have required programs: +# print STDERR "Inside burntool loop!\n"; + my $regtool = can_run('regtool') or &my_die ("Can't find regtool", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my $burntool = can_run('burntool') or &my_die ("Can't find burntool", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + + ## Check the current burntool processing version: + my $regtool_state_cmd = "$regtool -processedimfile -exp_id $exp_id -class_id $class_id -limit 1"; + if (defined($dbname)) { + $regtool_state_cmd .= " -dbname $dbname"; + } + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $regtool_state_cmd, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform regtool: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + my $regData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse regtool metadata", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my $regData2 = parse_md_list($regData); + + my $burntoolState = 999; + foreach my $regEntry (@$regData2) { +# print "$regEntry->{exp_id} $regEntry->{burntool_state}\n"; + + if ($regEntry->{exp_id} == $exp_id) { + $burntoolState = $regEntry->{burntool_state}; + } + } + if ($burntoolState == 999) { + &my_die("Unable to find burntool_state in metadata", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + if ($burntoolState == 32767) { + $burntoolState = 0; + } + + ## Read camera config to get the current good burntool state : + ## XXX This is extremely slow. Any better way to do this? + my $camera_config_cmd = "$ppConfigDump -camera $camera -dump-camera -"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $camera_config_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + my $camData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse ppConfigDump metadata", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + + my $burntoolStateGood; + foreach my $camEntry (@$camData) { + if ($camEntry->{name} eq "BURNTOOL.STATE.GOOD") { + $burntoolStateGood = $camEntry->{value}; + } + } + + if (abs($burntoolState) != $burntoolStateGood) { + if ($useBestBurntool) { + &my_die("Image burntool version does not match current accepted version.", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + else { + warn ("Image burntool version does not match current accepted version. Continuing as requested."); + # Internally pretend that we're at a good version. + if ($burntoolState < -10) { + $burntoolState = -1 * $burntoolStateGood; + } + elsif ($burntoolState > 10) { + $burntoolState = $burntoolStateGood; + } + elsif ($burntoolState == 0) { + # You've told me to use a deburned image, and that you don't care if it's the most recent. The database has told me + # that burntool has never been run on this image. We'll hope the database is wrong, but I don't really trust that. + warn ("burntool_state suggests no table will be found. This will likely crash."); + $burntoolState = -1 * $burntoolStateGood; + } + else { + &my_die("No valid burntool table will be found.", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + } + } + + ## We now know that we have an image that has been burntooled. + my ($tempFile, $tempName) = tempfile( "/tmp/chip.$exp_id.$class_id.deburned.XXXX", + UNLINK => !$save_temps, SUFFIX => '.fits' ); + + # get the UNIX version of the (possible) neb: or path: filename + my $uriReal = $ipprc->file_resolve( $uri ); + + # funpack into the temp file. + my $funpack_cmd = "$funpack -S $uriReal > $tempName"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + ## Construct commands to apply the pre-calculated burntool trailfits to the pixel data. + my ($burntoolTable_uri, $burntoolTable_uriReal); + my $burntool_cmd = "$burntool "; + + if ($burntoolState == -1 * $burntoolStateGood) { # Burntool information stored in an external table. + $burntoolTable_uri = $uri; + $burntoolTable_uri =~ s/fits$/burn.tbl/; + $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri ); + unless ($ipprc->file_exists($burntoolTable_uri)) { + &my_die("Couldn't find burntool table: $burntoolTable_uri",$exp_id,$chip_id,$class_id, $PS_EXIT_SYS_ERROR); + } + + $burntool_cmd .= "$tempName in=${burntoolTable_uriReal} apply=t"; + } + elsif ($burntoolState == $burntoolStateGood) { # Burntool information stored in a header table. + $burntool_cmd .= "$tempName apply=t"; + } + else { + &my_die("Image data not properly burntooled, impossible state", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + ## Run burntool to change the pixels of tempfile, and repoint $uri to that file. + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = + run(command => $burntool_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform burntool: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + $uri = $tempName; + unless ($ipprc->file_exists($uri)) { + &my_die("Couldn't find deburned input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + +# print STDERR "$uri $uriReal $tempName $burntoolTable_uri $burntoolTable_uriReal $burntool_cmd $save_temps\n"; +# exit(100); } @@ -152,48 +288,48 @@ if ($tiltystreakApply) { - my $tiltystreakByClass = metadataLookupMD($recipeData, 'TILTYSTREAK.BY.CLASS'); - my $tiltystreakClassApply = metadataLookupBool($tiltystreakByClass, "APPLY.$class_id"); - if ($tiltystreakClassApply) { - - my $tiltystreakClassOptions = metadataLookupStr($tiltystreakByClass, "OPTIONS.$class_id"); - if ($tiltystreakClassOptions eq "none") { - $tiltystreakClassOptions = ""; - } - - my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf); - - # XXX make these optional (must be built by psbuild as well...) - my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); - my $tiltystreak = can_run('tiltystreak') or &my_die ("Can't find tiltystreak", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); - - # create an temporary output file: - my ($tempFile, $tempName) = tempfile( "/tmp/chip.$exp_id.$class_id.tmp.XXXX", UNLINK => !$save_temps ); - - print "uri: $uri\n"; - - # get the UNIX version of the (possible) neb: or path: filename - my $uriReal = $ipprc->file_resolve( $uri ); - - print "uriReal: $uriReal\n"; - - # unpack the data (is a NOP if not compressed) - $command = "$funpack -S $uriReal > $tempName"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); - } - - # run tiltystreak - $command = "$tiltystreak $tempName $tiltystreakClassOptions"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform tiltystreak: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); - } - - # supply the output file as the new input file - $uri = $tempName; - } + my $tiltystreakByClass = metadataLookupMD($recipeData, 'TILTYSTREAK.BY.CLASS'); + my $tiltystreakClassApply = metadataLookupBool($tiltystreakByClass, "APPLY.$class_id"); + if ($tiltystreakClassApply) { + + my $tiltystreakClassOptions = metadataLookupStr($tiltystreakByClass, "OPTIONS.$class_id"); + if ($tiltystreakClassOptions eq "none") { + $tiltystreakClassOptions = ""; + } + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf); + + # XXX make these optional (must be built by psbuild as well...) + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my $tiltystreak = can_run('tiltystreak') or &my_die ("Can't find tiltystreak", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + + # create an temporary output file: + my ($tempFile, $tempName) = tempfile( "/tmp/chip.$exp_id.$class_id.tmp.XXXX", UNLINK => !$save_temps ); + + print "uri: $uri\n"; + + # get the UNIX version of the (possible) neb: or path: filename + my $uriReal = $ipprc->file_resolve( $uri ); + + print "uriReal: $uriReal\n"; + + # unpack the data (is a NOP if not compressed) + $command = "$funpack -S $uriReal > $tempName"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + # run tiltystreak + $command = "$tiltystreak $tempName $tiltystreakClassOptions"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform tiltystreak: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + # supply the output file as the new input file + $uri = $tempName; + } } @@ -253,21 +389,5 @@ } chomp $cmdflags; - if (($camera ne "GPC1") or ($class_id ne "XY27") or ($exp_id < 53171)) { - ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag - } else { - # hack to kick ota 27 out of subsequent analysis by setting quality flag to a non-zero value - my ($before, $after) = split " -quality ", $cmdflags; - - # get the current value - ($quality) = $after =~ /^(\d+)/; - $after = substr($after, length($quality)); - - # replace it if it is zero - $quality = 42 if !$quality; - - # rebuild the cmdflags - $cmdflags = $before . " -quality $quality"; - $cmdflags .= " $after" if $after; - } + ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag } Index: branches/eam_branches/20090820/ippScripts/scripts/dist_component.pl =================================================================== --- branches/eam_branches/20090820/ippScripts/scripts/dist_component.pl (revision 25206) +++ branches/eam_branches/20090820/ippScripts/scripts/dist_component.pl (revision 25766) @@ -164,4 +164,8 @@ or ($file_rule =~ /.JPEG2/)); + if ($stage eq "diff") { + next if $file_rule =~ /CONV/; + } + my $file_name = $file->{name}; my $base = basename($file_name); Index: branches/eam_branches/20090820/ippScripts/scripts/dist_make_fileset.pl =================================================================== --- branches/eam_branches/20090820/ippScripts/scripts/dist_make_fileset.pl (revision 25206) +++ branches/eam_branches/20090820/ippScripts/scripts/dist_make_fileset.pl (revision 25766) @@ -39,5 +39,6 @@ # Parse the command-line arguments -my ($dist_id, $dist_dir, $target_id, $stage, $stage_id, $prod_id, $product_name, $ds_dbhost, $ds_dbname); +my ($dist_id, $dist_dir, $target_id, $stage, $stage_id, $dest_id, $product_name, $ds_dbhost, $ds_dbname); +my ($label, $filter); my ($dbname, $save_temps, $verbose, $no_update, $logfile); @@ -48,6 +49,8 @@ 'stage=s' => \$stage, # raw, chip, camera, fake, warp, stack, or diff 'stage_id=s' => \$stage_id, # exp_id, chip_id, etc. - 'prod_id=s' => \$prod_id, # id for the product + 'dest_id=s' => \$dest_id, # id for the product 'product_name=s' => \$product_name, # location of the data store directory for this product + 'label=s' => \$label, + 'filter=s' => \$filter, 'ds_dbhost=s' => \$ds_dbhost, # database host for the datastore database 'ds_dbname=s' => \$ds_dbname, # database name for the datastore database @@ -60,5 +63,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --dist_id --dist_dir --target_id --stage --stage_id --prod_id --ds_dbhost --ds_dbname", +pod2usage( -msg => "Required options: --dist_id --dist_dir --target_id --stage --stage_id --label --filter --dest_id --ds_dbhost --ds_dbname", -exitval => 3) unless defined $dist_id and @@ -67,5 +70,7 @@ defined $stage and defined $stage_id and - defined $prod_id and + defined $label and + defined $filter and + defined $dest_id and defined $product_name and defined $ds_dbhost and @@ -79,7 +84,8 @@ my $fs_tag = get_fileset_tag($ipprc, $stage, $stage_id, $dbname); -&my_die("failed to lookup fileset tag", $dist_id, $prod_id, $PS_EXIT_UNKNOWN_ERROR) if !$fs_tag; - -my $fileset_name = "$fs_tag.$stage.$stage_id.$dist_id.$prod_id"; +&my_die("failed to lookup fileset tag", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR) if !defined $fs_tag; + +my $fileset_name = $fs_tag ? "$fs_tag." : ""; +$fileset_name .= "$stage.$stage_id.$dist_id.$dest_id"; print "$fileset_name\n"; @@ -89,5 +95,5 @@ my $dbinfo_file = "dbinfo.$stage.$stage_id.mdc"; if (! -e "$dist_dir/$dbinfo_file" ) { - &my_die("dbinfo file for dist run $dbinfo_file not found", $dist_id, $prod_id, $PS_EXIT_UNKNOWN_ERROR); + &my_die("dbinfo file for dist run $dbinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR); } print "dbinfo file $dbinfo_file exists\n" if $verbose; @@ -96,5 +102,5 @@ my $dirinfo_file = "dirinfo.$stage.$stage_id.mdc"; if (! -e "$dist_dir/$dirinfo_file" ) { - &my_die("dirinfo file for dist run $dirinfo_file not found", $dist_id, $prod_id, $PS_EXIT_UNKNOWN_ERROR); + &my_die("dirinfo file for dist run $dirinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR); } print "dirinfo file $dirinfo_file exists\n" if $verbose; @@ -116,9 +122,9 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform $command error_code: $error_code", $dist_id, $prod_id, $error_code); + &my_die("Unable to perform $command error_code: $error_code", $dist_id, $dest_id, $error_code); } my $metadata = $mdcParser->parse (join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $dist_id, $prod_id, $PS_EXIT_PROG_ERROR); + &my_die("Unable to parse metadata config doc", $dist_id, $dest_id, $PS_EXIT_PROG_ERROR); $components = parse_md_list($metadata); @@ -146,6 +152,5 @@ { - # XXX: need to chose an appropriate file set type - my $command = "$dsreg --add $fileset_name --product $product_name --type notset --list $listFileName"; + my $command = "$dsreg --add $fileset_name --product $product_name --type IPP-DIST --list $listFileName"; # the data store will refer to the distribution bundle via symlinks back to distRun.outdir @@ -153,18 +158,21 @@ # set the product specific columns in product list - $command .= " --ps0 $target_id --ps1 $stage --ps2 $stage_id --ps3 $fs_tag"; - - $command .= " --dbname $ds_dbname"; # XXX: notyet --dbhost $ds_dbhost - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform $command error_code: $error_code", $dist_id, $prod_id, $error_code); + my $prod_col_3 = $fs_tag ? $fs_tag : "$stage.$stage_id"; + + $command .= " --ps0 $target_id --ps1 $stage --ps2 $stage_id --ps3 $prod_col_3"; + $command .= " --ps4 $label --ps5 $filter"; + + $command .= " --dbname $ds_dbname --dbhost $ds_dbhost"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command error_code: $error_code", $dist_id, $dest_id, $error_code); } } { - my $command = "$disttool -addfileset -dist_id $dist_id -prod_id $prod_id -name $fileset_name"; + my $command = "$disttool -addfileset -dist_id $dist_id -dest_id $dest_id -name $fileset_name"; $command .= " -dbname $dbname" if $dbname; @@ -175,5 +183,5 @@ # We need to have revertfileset check whether the fileset exists and do dsreg -del if it does $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform $command error_code: $error_code", $dist_id, $prod_id, $error_code); + &my_die("Unable to perform $command error_code: $error_code", $dist_id, $dest_id, $error_code); } } @@ -211,8 +219,6 @@ my $dbname = shift; - if ($stage eq 'stack') { - return "stack.$stage_id"; - } elsif ($stage eq 'diff') { - return "diff.$stage_id"; + if (($stage eq 'stack') or ($stage eq 'diff')) { + return ""; } @@ -248,5 +254,5 @@ my $tag = $ref->{exp_name}; - return $tag; + return "$tag"; } @@ -254,11 +260,11 @@ my $msg = shift; my $dist_id = shift; - my $prod_id = shift; + my $dest_id = shift; my $fault = shift; - # TODO: disttool -adddsfileset -prod_id $prod_id -dist_id $dist_id -fault $fault + # TODO: disttool -adddsfileset -dest_id $dest_id -dist_id $dist_id -fault $fault print STDERR "$msg\n"; - my $command = "$disttool -addfileset -dist_id $dist_id -prod_id $prod_id -fault $fault"; + my $command = "$disttool -addfileset -dist_id $dist_id -dest_id $dest_id -fault $fault"; $command .= " -dbname $dbname" if $dbname; Index: branches/eam_branches/20090820/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- branches/eam_branches/20090820/ippScripts/scripts/ipp_cleanup.pl (revision 25206) +++ branches/eam_branches/20090820/ippScripts/scripts/ipp_cleanup.pl (revision 25766) @@ -60,8 +60,8 @@ -my %stages = ( chip => 1, camera => 1, fake => 1, warp => 1, stack => 1, diff => 1, - detrend.process.imfile => 1, detrend.process.exp => 1, detrend.stack.imfile => 1, - detrend.normstat.imfile => 1, detrend.norm.imfile => 1, detrend.norm.exp => 1, - detrend.resid.imfile => 1, detrend.resid.exp => 1 ); +my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff" => 1, + "detrend.process.imfile" => 1, "detrend.process.exp" => 1, "detrend.stack.imfile" => 1, + "detrend.normstat.imfile" => 1, "detrend.norm.imfile" => 1, "detrend.norm.exp" => 1, + "detrend.resid.imfile" => 1, "detrend.resid.exp" => 1 ); unless ($stages{$stage}) { die "unknown stage $stage for ipp_cleanup.pl\n"; @@ -119,13 +119,24 @@ # don't clean up unless the data needed to update is available # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); - print STDERR "CHIP: CONFIG_FILE : $config_file\n"; - if (!$config_file or ! -e $config_file) { + +# if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for chipRun $stage_id $class_id " - . " because config file is missing\n"; + . " because config file ($config_file) is missing\n"; $status = 0; } } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping scrubbed for chipRun $stage_id $class_id " + . " because config file ($config_file) is present\n"; + $status = 0; + } + } if ($status) { @@ -134,7 +145,7 @@ # delete the temporary image datafiles - addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id); +# addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); +# addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); +# addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id); addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id); addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id); @@ -144,9 +155,9 @@ addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id); addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); # clean? + addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); # clean? addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id); addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); #clean? addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id); } @@ -160,7 +171,12 @@ if ($mode eq "goto_purged") { $command .= " -topurgedimfile"; - } else { + } + elsif ($mode eq "goto_cleaned") { $command .= " -tocleanedimfile"; } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedimfile"; + } + $command .= " -dbname $dbname" if defined $dbname; @@ -218,16 +234,25 @@ my $status = 1; # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); - - if (!$config_file or ! -e $config_file) { + + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for camRun $stage_id because config file is missing\n"; $status = 0; } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for camRun $stage_id because config file ($config_file) is present\n"; + $status = 0; + } } if ($status) { my @files = (); # delete the temporary image datafiles - addFilename (\@files, "PSASTRO.OUTPUT", $path_base); +# addFilename (\@files, "PSASTRO.OUTPUT", $path_base); if ($mode eq "goto_purged") { # additional files to remove for 'purge' mode @@ -246,5 +271,5 @@ } if ($mode eq "goto_scrubbed") { - $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned"; + $command = "$camtool -updaterun -cam_id $stage_id -set_state scrubbed"; } if ($mode eq "goto_purged") { @@ -305,5 +330,5 @@ my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); - if (!$config_file or ! -e $config_file) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" . " because config file is missing\n"; @@ -311,24 +336,36 @@ } } - if ($status) { - # delete the temporary image datafiles - addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id ); - addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); - addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id); - addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id); - - if ($mode eq "goto_purged") { - # additional files to remove for 'purge' mode - addFilename(\@files, "PSWARP.BIN1", $path_base, $skycell_id ); - addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id ); - addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id ); - # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base); - - # XXX: do we want to delete these? - # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id); - # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id); - } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping scrubbed for warpRun $stage_id $skycell_id" . + " because config file is present\n"; + $status = 0; + } + } + if ($status) { + if ($skyfile->{quality} != 8007) { + my @files = (); + + # delete the temporary image datafiles + addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id ); + addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); + addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id); +# addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename(\@files, "PSWARP.BIN1", $path_base, $skycell_id ); + addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id ); + addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id ); + # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base); + + # XXX: do we want to delete these? + # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id); + # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id); + } # actual command to delete the files - $status = &delete_files (\@files); + $status = &delete_files (\@files); + } } @@ -337,7 +374,11 @@ if ($mode eq "goto_purged") { $command .= " -topurgedskyfile"; - } else { + } + elsif ($mode eq "goto_cleaned") { $command .= " -tocleanedskyfile"; } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedskyfile"; + } $command .= " -dbname $dbname" if defined $dbname; @@ -395,15 +436,22 @@ if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); - print STDERR "MY CONFIG FILE = $config_file\n"; - printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file); - $config_file =~ s%^file://%%; - if (!$config_file or ! -e $config_file) { + + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for stackRun $stage_id $skycell_id" . " because config file is missing\n"; $status = 0; } - $config_file = 'file://' . $config_file; + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" . + " because config file is present\n"; + $status = 0; + } } if ($status) { + my @files = (); # delete the temporary image datafiles addFilename(\@files, "PPSTACK.OUTPUT", $path_base, $skycell_id); @@ -428,6 +476,10 @@ if ($mode eq "goto_purged") { $command .= " -updaterun -state purged"; - } else { + } + elsif ($mode eq "goto_cleaned") { $command .= " -updaterun -state cleaned"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -updaterun -state scrubbed"; } $command .= " -dbname $dbname" if defined $dbname; @@ -485,15 +537,22 @@ if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); - print STDERR "MY CONFIG FILE = $config_file\n"; - printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file); - $config_file =~ s%^file://%%; - if (!$config_file or ! -e $config_file) { + + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" . " because config file ($config_file) is missing\n"; $status = 0; } - $config_file = 'file://' . $config_file; + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" . + " because config file ($config_file) is present\n"; + $status = 0; + } } if ($status) { + my @files = (); # delete the temporary image datafiles addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id); @@ -504,4 +563,5 @@ addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id); addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id); + addFilename(\@files, "PPSUB.INPUT.CONV", $path_base, $skycell_id); addFilename(\@files, "PPSUB.INPUT.CONV.MASK", $path_base, $skycell_id); @@ -523,16 +583,21 @@ } - print STDERR "MY FILES: @files\n"; +# print STDERR "MY FILES: @files\n"; $status = &delete_files(\@files); } - print STDERR "MY STATUS: $status\n"; +# print STDERR "MY STATUS: $status\n"; if ($status) { my $command = "$difftool -diff_id $stage_id"; -# my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id"; + if ($mode eq "goto_purged") { $command .= " -updaterun -state purged"; - } else { + } + elsif ($mode eq "goto_cleaned") { $command .= " -updaterun -state cleaned"; } + elsif ($mode eq "goto_scrubbed") { + $command .= " -updaterun -state scrubbed"; + } + $command .= " -dbname $dbname" if defined $dbname; @@ -545,5 +610,5 @@ } else { my $command = "$difftool -updaterun -diff_id $stage_id -state $error_state"; -# my $command = "$difftool -updaterun -diff_id $stage_id -skycell_id $skycell_id -state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; @@ -560,7 +625,1208 @@ } if ($stage eq 'fake') { - die "ipp_cleanup.pl -stage fake not yet implemented. Probably will just mark database cleaned.\n"; + print STDERR "This does not seem to work at present, as no files exist. Terminating quietly.\n"; + exit(0); + die "--stage_id required for stage fake\n" if !$stage_id; + ### select the imfiles for this entry + + # this stage uses 'chiptool' + my $faketool = can_run('faketool') or die "Can't find faketool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $imfiles; # Array of component files + my $command = "$faketool -pendingcleanupimfile -fake_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); + } + + # if there are no fakeProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' + if (@$stdout_buf == 0) { + my $command = "$faketool -fake_id $stage_id -updaterun -set_state new"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); + } + exit 0; + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "fake", $stage_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "fake", $stage_id, $PS_EXIT_PROG_ERROR); + + # loop over all of the imfiles, determine the path_base and class_id for each + foreach my $imfile (@$imfiles) { + my $class_id = $imfile->{class_id}; + my $path_base = $imfile->{path_base}; + my $status = 1; + + # don't clean up unless the data needed to update is available + # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); + + unless ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for fakeRun $stage_id $class_id " + . " because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping scrubbed for fakeRun $stage_id $class_id " + . " because config file is present\n"; + $status = 0; + } + } + + if ($status) { + # array of actual filenames to delete + my @files = (); + + # delete the temporary image datafiles + addFilename (\@files, "PPSIM.OUTPUT.MEF", $path_base, $class_id); + addFilename (\@files, "PPSIM.OUTPUT.SPL", $path_base, $class_id); + addFilename (\@files, "PPSIM.FAKE.CHIP", $path_base, $class_id); + addFilename (\@files, "PPSIM.FORCE.CHIP", $path_base, $class_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPSIM.SOURCES", $path_base, $class_id); + addFilename (\@files, "PPSIM.FAKE.SOURCES", $path_base, $class_id); + addFilename (\@files, "PPSIM.FORCE.SOURCES", $path_base, $class_id); + } + + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$faketool -fake_id $stage_id -class_id $class_id"; + if ($mode eq "goto_purged") { + $command .= " -topurgedimfile"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -tocleanedimfile"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedimfile"; + } + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); + } + } else { + + # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) + my $command = "$faketool -updateprocessedimfile -fake_id $stage_id -class_id $class_id -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); + } + } + } + exit 0; + } -# fake : faketool : -pendingcleanupimfile (loop over imfiles) +# Detrend stages +if ($stage eq "detrend.process.imfile") { + + die "--stage_id required for stage detrend.process.imfile\n" if !$stage_id; + ### select the imfiles for this entry + + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find chiptool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $imfiles; # Array of component files + my $command = "$dettool -pendingcleanup_processedimfile -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code); + } + + # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' + if (@$stdout_buf == 0) { + exit 0; # Silently exit if there's nothing to do. I don't know how we'd ever get here, but let's be safe. + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + # loop over all of the imfiles, determine the path_base and class_id for each + foreach my $imfile (@$imfiles) { + my $exp_id = $imfile->{exp_id}; + my $class_id = $imfile->{class_id}; + my $path_base = $imfile->{path_base}; + my $status = 1; + + # don't clean up unless the data needed to update is available + # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. + + # Possibly not the correct config file, but simtest doesn't leave any around to check. + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + unless ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for detrend.process.imfile $stage_id $class_id " + . " because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping scrubbed for detrend.process.imfile $stage_id $class_id " + . " because config file is present\n"; + $status = 0; + } + } + + if ($status) { + # array of actual filenames to delete + my @files = (); + + # delete the temporary image datafiles + addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); + } + + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -det_id $stage_id -exp_id $exp_id -class_id $class_id -updateprocessedimfile"; + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + + } else { + + # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) + my $command = "$dettool -updateprocessedimfile -det_id $stage_id -exp_id $exp_id -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } + } + + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; +} +if ($stage eq "detrend.process.exp") { + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $exps; # Array of component files + my $command = "$dettool -pendingcleanup_processedexp -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + exit 0; #silently abort. I need to fix this for propers + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + + foreach my $exp (@$exps) { + my $path_base = $exp->{path_base}; + my $exp_id = $exp->{exp_id}; + + my $status = 1; + # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base); + + unless ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + # delete the temporary image datafiles + # I can't find anything to put here + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.JPEG1", $path_base); + addFilename (\@files, "PPIMAGE.JPEG2", $path_base); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + # since 'camera' has only a single imfile, we can just update the run + my $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; +} +if ($stage eq "detrend.stack.imfile") { + + die "--stage_id required for stage $stage\n" if !$stage_id; + + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + my $stacks; # Array reference of component files + my $command = "$dettool -pendingcleanup_stacked -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $stacks = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + my @files = (); + foreach my $stack (@{ $stacks }) { + # detStackedImfile does not have a path_base column. This is inconvenient, as it means we need to calculate it. + my $path_base = $stack->{uri}; + my $iteration = $stack->{iteration}; + my $class_id = $stack->{class_id}; + + $path_base =~ s/\.fits$//; # That should do it? + + my $status = 1; + +# if ($mode eq "goto_cleaned") { +# my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id); + +# $config_file =~ s%^file://%%; +# if (!$config_file or ! -e $config_file) { +# print STDERR "skipping cleanup for $stage $stage_id $path_base" . +# " because config file is missing\n"; +# $status = 0; +# } +# $config_file = 'file://' . $config_file; +# } +# elsif ($mode eq "goto_scrubbed") { +# my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id); +# $config_file =~ s%^file://%%; +# if ($config_file and -e $config_file) { +# print STDERR "skipping scrubbed for $stage $stage_id $path_base" . +# " because config file is present\n"; +# $status = 0; +# } +# } + + if ($status) { + my @files = (); + # delete the temporary image datafiles + # There's no convenient way to get the detrend type, so I'm queueing all of them for deletion. + # I understand that they all point to the same filename right now, but that may not be true in + # the future. + addFilename(\@files, "PPMERGE.OUTPUT.MASK", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.BIAS", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.DARK", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.SHUTTER", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.FLAT", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.FRINGE", $path_base, $stage_id); + + + addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $stage_id); + + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode +# addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $stage_id); + } + + $status = &delete_files(\@files); + } + + if ($status) { + my $command = "$dettool -det_id $stage_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_purged") { + $command .= " -updatestacked -data_state purged"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -updatestacked -data_state cleaned"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -updatestacked -data_state scrubbed"; + } + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatestacked -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + exit 0; +} +if ($stage eq "detrend.normstat.imfile") { + print STDERR "I'm not convinced there's anything to clean up from stage $stage\n"; + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'camtool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + my $command = "$dettool -pendingcleanup_normalizedstat -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { +# my $path_base = $exp->{path_base}; + my $iteration = $exp->{iteration}; + my $class_id = $exp->{class_id}; + + my $status = 1; + if ($status) { + my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; + +} +if ($stage eq "detrend.norm.imfile") { + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $exps; # Array of component files + my $command = "$dettool -pendingcleanup_normalizedimfile -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $path_base = $exp->{path_base}; + my $iteration = $exp->{iteration}; + my $class_id = $exp->{class_id}; + + my $status = 1; + # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + unless ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base); + addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base); + + addFilename (\@files, "PPIMAGE.OUTPUT", $path_base); + addFilename (\@files, "PPIMAGE.STATS", $path_base); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; +} +if ($stage eq "detrend.norm.exp") { + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $exps; # Array of component files + my $command = "$dettool -pendingcleanup_normalizedexp -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + exit 0; + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $exp_id = $exp->{exp_id}; + my $iteration = $exp->{iteration}; + my $path_base = $exp->{path_base}; + + my $status = 1; + # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + unless ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + # delete the temporary image datafiles + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.JPEG1", $path_base); + addFilename (\@files, "PPIMAGE.JPEG2", $path_base); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -updatenormalizedexp -det_id $stage_id -iteration $iteration"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR " residexp had an issue setting the state:? $success $error_code\n"; + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatenormalizedexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + exit 0; +} +if ($stage eq "detrend.resid.imfile") { + + die "--stage_id required for stage $stage\n" if !$stage_id; + ### select the imfiles for this entry + + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $imfiles; # Array of component files + my $command = "$dettool -pendingcleanup_residimfile -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + exit 0; + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + # loop over all of the imfiles, determine the path_base and class_id for each + foreach my $imfile (@$imfiles) { + my $class_id = $imfile->{class_id}; + my $iteration = $imfile->{iteration}; + my $exp_id = $imfile->{exp_id}; + my $path_base = $imfile->{path_base}; + my $status = 1; + + # don't clean up unless the data needed to update is available + # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. + + # Possibly not the correct config file, but simtest doesn't leave any around to check. + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + unless ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for $stage $stage_id $class_id " + . " because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping scrubbed for $stage $stage_id $class_id " + . " because config file is present\n"; + $status = 0; + } + } + + if ($status) { + # array of actual filenames to delete + my @files = (); + + # delete the temporary image datafiles + addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); + } + + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; +} +if ($stage eq "detrend.resid.exp") { + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'camtool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $exps; # Array of component files + my $command = "$dettool -pendingcleanup_residexp -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + # This is a hack to bomb out until I can diagnose why pantasks wants to keep running this + if (@$stdout_buf == 0) { + exit 0; + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $exp_id = $exp->{exp_id}; + my $iteration = $exp->{iteration}; + my $path_base = $exp->{path_base}; +# my $class_id = $exp->{class_id} + my $status = 1; + # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + unless ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + # delete the temporary image datafiles + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.JPEG1", $path_base);#, $class_id); + addFilename (\@files, "PPIMAGE.JPEG2", $path_base);#, $class_id); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR " residexp had an issue setting the state:? $success $error_code\n"; + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + exit 0; +} + die "ipp_cleanup.pl -stage $stage not yet implemented\n"; @@ -569,10 +1835,37 @@ { my $files = shift; # reference to a list of files to unlink +# my $test_verbose = 1; + +# if ($test_verbose == 1) { +# open(TMPLOG,">>/tmp/czw.cleanup.log"); +# flock(TMPLOG,2); +# } # this script is, of course, very dangerous. foreach my $file (@$files) { - print STDERR "unlinking $file\n"; + print STDERR "unlinking $stage $stage_id $file"; + unless ($ipprc->file_exists($file)) { + print STDERR "\t File not found\n"; + } + else { + print STDERR "\n"; + } + +# if ($test_verbose == 1) { +# print TMPLOG "$stage $stage_id $file"; + +# else { +# print TMPLOG "\n"; +# } +# } + $ipprc->file_delete($file); } + +# if ($test_verbose == 1) { +# flock(TMPLOG,8); +# close(TMPLOG); +# } + return 1; } Index: branches/eam_branches/20090820/ippScripts/scripts/ipp_image_path.pl =================================================================== --- branches/eam_branches/20090820/ippScripts/scripts/ipp_image_path.pl (revision 25206) +++ branches/eam_branches/20090820/ippScripts/scripts/ipp_image_path.pl (revision 25766) @@ -108,7 +108,7 @@ # remove the leading "file://" if present $path =~ s/^file\:\/\///; + $nfiles++; + print "$path\n"; } - $nfiles++; - print "$path\n"; } if (!$nfiles) { Index: branches/eam_branches/20090820/ippScripts/scripts/magic_destreak_revert.pl =================================================================== --- branches/eam_branches/20090820/ippScripts/scripts/magic_destreak_revert.pl (revision 25206) +++ branches/eam_branches/20090820/ippScripts/scripts/magic_destreak_revert.pl (revision 25766) @@ -191,7 +191,7 @@ } -revert_files($image, $mask, $weight, $sources, $astrom, $bimage, $bmask, $bweight, $bsources, $bastrom); - -if ($stage eq "diff") { +revert_files($replace, $image, $mask, $weight, $sources, $astrom, $bimage, $bmask, $bweight, $bsources, $bastrom); + +if ($stage eq "diff" and $inverse) { my $name = "PPSUB.INVERSE"; $image = $ipprc->filename($name, $path_base); @@ -203,10 +203,10 @@ $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base); $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base); - revert_files($image, $mask, $weight, $sources, undef, $bimage, $bmask, $bweight, $bsources, undef); + revert_files($replace, $image, $mask, $weight, $sources, undef, $bimage, $bmask, $bweight, $bsources, undef); } # now revert the row in the database { - my $command = "$magicdstool -revertdestreakedfile"; + my $command = "$magicdstool -revertdestreakedfile -i_am_sure"; $command .= " -magic_ds_id $magic_ds_id"; $command .= " -component $component"; @@ -231,4 +231,7 @@ sub revert_files { + my $replace = shift; + return if !$replace; + my $image = shift; my $mask = shift; Index: branches/eam_branches/20090820/ippScripts/scripts/publish_file.pl =================================================================== --- branches/eam_branches/20090820/ippScripts/scripts/publish_file.pl (revision 25206) +++ branches/eam_branches/20090820/ippScripts/scripts/publish_file.pl (revision 25766) @@ -39,6 +39,6 @@ # Parse the command-line arguments -my ( $pub_id, $camera, $stage, $stage_id, $format, $product, $workdir ); -my ( $dbname, $verbose, $no_update, $save_temps, $redirect ); +my ( $pub_id, $camera, $stage, $stage_id, $fileset, $format, $product, $workdir ); +my ( $dbname, $verbose, $no_update, $no_op, $save_temps, $redirect ); GetOptions( @@ -48,8 +48,10 @@ 'stage_id=s' => \$stage_id, # Stage identifier 'product=s' => \$product, # Datastore product name + 'fileset=s' => \$fileset, # Fileset name 'workdir=s' => \$workdir, # Working directory 'dbname=s' => \$dbname, # Database name 'verbose' => \$verbose, # Print to stdout 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations 'save-temps' => \$save_temps, # Save temporary files? 'redirect-output' => \$redirect, # Redirect output to log file? @@ -77,23 +79,9 @@ my $mdcParser = PS::IPP::Metadata::Config->new; - -# Retrieve file name of interest -my %files; # Input filenames -my %zp; # Zero points -my %exp_id; # Exposure identifiers -my %exp_name; # Exposure names -my %direction; # Direction of subtraction -{ - my $command; # Command to run - - if ($stage eq 'diff') { - $command = "difftool -diffskyfile -diff_id $stage_id"; - } elsif ($stage eq 'camera') { - $command = "camtool -processedexp -cam_id $stage_id"; - } else { - &my_die( "Unrecognised stage: $stage", $pub_id, $PS_EXIT_CONFIG_ERROR ); - } +my ($dsFile, $dsFileName) = tempfile("/tmp/publish.$pub_id.ds.XXXX", UNLINK => !$save_temps ); + +if ($stage eq 'camera') { + my $command = "camtool -processedexp -cam_id $stage_id"; $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -106,4 +94,31 @@ &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR); + &my_die("More than one entry for cam_id $stage_id", $pub_id, $PS_EXIT_PROG_ERROR) if scalar @$components > 1; + + my $comp = $$components[0]; # Component of interest + my $path_base = $comp->{path_base}; # Base name for file + my $file = $ipprc->filename( "PSASTRO.OUTPUT", $path_base ); + $file = $ipprc->file_resolve($file); + my $cam_id = $comp->{cam_id}; + my $name = "cam_$cam_id"; + print $dsFile "$file|||$product|$name|\n"; +} elsif ($stage eq 'diff') { + my $command = "difftool -diffskyfile -diff_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $success; + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR); + + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR); + + my ($mopsPositiveFile, $mopsPositiveFileName) = tempfile("/tmp/publish.$pub_id.mops.pos.XXXX", UNLINK => !$save_temps ) if $product eq 'IPP-MOPS'; + my ($mopsNegativeFile, $mopsNegativeFileName) = tempfile("/tmp/publish.$pub_id.mops.neg.XXXX", UNLINK => !$save_temps ) if $product eq 'IPP-MOPS'; + + my %positive; # Data for positive diff detections + my %negative; # Data for negative diff detections foreach my $comp ( @$components ) { my $path_base = $comp->{path_base}; # Base name for file @@ -112,69 +127,78 @@ (carp "Bad zpt_obs or exp_time for component" and next) if not defined $comp->{zpt_obs} or not defined $comp->{exp_time}; my $zp = $comp->{zpt_obs} + 2.5 * log($comp->{exp_time}) / log(10); - - if ($stage eq 'diff') { - my $skycell_id = $comp->{skycell_id}; - - # Positive direction - $files{"$skycell_id.pos"} = $ipprc->filename( "PPSUB.OUTPUT.SOURCES", $path_base ); - $zp{"$skycell_id.pos"} = $zp; - $exp_id{"$skycell_id.pos"} = $comp->{exp_id_1}; - $exp_name{"$skycell_id.pos"} = $comp->{exp_name_1}; - $direction{"$skycell_id.pos"} = 1; - - # Negative direction - if (defined $comp->{bothways} and $comp->{bothways}) { - $files{"$skycell_id.neg"} = $ipprc->filename( "PPSUB.INVERSE.SOURCES", $path_base ); - $zp{"$skycell_id.neg"} = $zp; - $exp_id{"$skycell_id.neg"} = $comp->{exp_id_2}; - $direction{"$skycell_id.neg"} = 0; - $exp_name{"$skycell_id.neg"} = $comp->{exp_name_2}; + my $astrom = sqrt($comp->{sigma_ra_1}**2 + $comp->{sigma_dec_1}**2); + + my $skycell_id = $comp->{skycell_id}; + my $filename = $ipprc->filename( "PPSUB.OUTPUT.SOURCES", $path_base ); + $filename = $ipprc->file_resolve($filename); + + my $data = { zp => $zp, + zp_err => $comp->{zpt_stdev}, + astrom => sqrt($comp->{sigma_ra_1}**2 + $comp->{sigma_dec_1}**2), + exp_name => $comp->{exp_name_1}, + exp_id => $comp->{exp_id_1}, + chip_id => $comp->{chip_id_1}, + cam_id => $comp->{cam_id_1}, + fake_id => $comp->{fake_id_1}, + warp_id => $comp->{warp1}, + diff_id => $comp->{diff_id}, + direction => 1, + }; + + diff_check(\%positive, $data, "positive"); + + if ($product eq 'IPP-MOPS') { + print $mopsPositiveFile "$filename\n"; + } else { + print $dsFile "$filename|||$product|${skycell_id}.pos|\n"; + } + + # Negative direction + if (defined $comp->{bothways} and $comp->{bothways}) { + my $filename = $ipprc->filename( "PPSUB.INVERSE.SOURCES", $path_base ); + $filename = $ipprc->file_resolve($filename); + + my $data = { zp => $zp, + zp_err => $comp->{zpt_stdev}, + astrom => sqrt($comp->{sigma_ra_2}**2 + $comp->{sigma_dec_2}**2), + exp_name => $comp->{exp_name_2}, + exp_id => $comp->{exp_id_2}, + chip_id => $comp->{chip_id_2}, + cam_id => $comp->{cam_id_2}, + fake_id => $comp->{fake_id_2}, + warp_id => $comp->{warp2}, + diff_id => $comp->{diff_id}, + direction => 0, + }; + + diff_check(\%negative, $data, "negative"); + + if ($product eq 'IPP-MOPS') { + print $mopsNegativeFile "$filename\n"; + } else { + print $dsFile "$filename|||$product|${skycell_id}.neg|\n"; } - - } elsif ($stage eq 'camera') { - my $cam_id = $comp->{cam_id}; - $files{$cam_id} = $ipprc->filename( "PSASTRO.OUTPUT", $path_base ); - $zp{$cam_id} = $zp; - $exp_id{$cam_id} = $comp->{exp_id}; - $exp_name{$cam_id} = $comp->{exp_name}; - $direction{$cam_id} = 1; } } -} - -# Prepare for data store input -my ($listFile, $listFileName) = tempfile("/tmp/publish.$pub_id.list.XXXX", UNLINK => !$save_temps ); - -# Process each file -foreach my $comp ( keys %files ) { - my $file = $ipprc->file_resolve( $files{$comp} ) or - &my_die("Unable to resolve file $files{$comp}", $pub_id, $PS_EXIT_SYS_ERROR); - my_die("Unable to find file $file", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $file ); - - my $zp = $zp{$comp}; - my $exp_id = $exp_id{$comp}; - my $exp_name = $exp_name{$comp}; - my $direction = $direction{$comp}; - if ($product eq "IPP-MOPS") { - my $outuri = "$outroot.$comp.fits"; - my $out = $ipprc->file_resolve( $outuri, 'create' ) or - &my_die( "Unable to resolve output file $outuri", $pub_id, $PS_EXIT_SYS_ERROR); - - my $command = "$ppMops $file $zp $exp_id $exp_name $direction $out"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - &my_die( "Unable to translate $file", $pub_id, $PS_EXIT_SYS_ERROR) unless $success; - &my_die( "Unable to find translated file $out", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $out ); - $file = $out; + + close $mopsPositiveFile; + close $mopsNegativeFile; + + if ($product eq 'IPP-MOPS') { + if (scalar keys %positive > 0) { + my $output = mops_combine(\%positive, "$outroot.pos.mops", $mopsPositiveFileName); + print $dsFile "$output|||$product|positive|\n"; + } + if (scalar keys %negative > 0) { + my $output = mops_combine(\%negative, "$outroot.neg.mops", $mopsNegativeFileName); + print $dsFile "$output|||$product|negative|\n"; + } } - - # format: filename|filesize|md5sum|filetype| - # note: since we omit filesize and md5sum, dsreg will calculate them - print $listFile "$file|||$product|$comp|\n"; -} - +} + +close $dsFile; unless ($no_update) { - my $command = "$dsreg --add $stage.$stage_id --copy --abspath --product $product --type $product --list $listFileName"; + my $command = "$dsreg --add $stage.$stage_id --copy --abspath --product $product --type $product --list $dsFileName"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -185,4 +209,7 @@ unless ($no_update) { my $command = "$pubtool -add -pub_id $pub_id -path_base $outroot"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; @@ -195,4 +222,73 @@ ### Pau. + +# Check inputs for a diff +sub diff_check +{ + my $data = shift; # Data hash + my $comp = shift; # Component data + my $name = shift; # Name of component + + $data->{zp} = $comp->{zp} unless defined $data->{zp}; + $data->{zp_err} = $comp->{zp_err} unless defined $data->{zp_err}; + $data->{astrom} = $comp->{astrom} unless defined $data->{astrom}; + $data->{exp_name} = $comp->{exp_name} unless defined $data->{exp_name}; + $data->{exp_id} = $comp->{exp_id} unless defined $data->{exp_id}; + $data->{chip_id} = $comp->{chip_id} unless defined $data->{chip_id}; + $data->{cam_id} = $comp->{cam_id} unless defined $data->{cam_id}; + $data->{fake_id} = $comp->{fake_id} unless defined $data->{fake_id}; + $data->{warp_id} = $comp->{warp_id} unless defined $data->{warp_id}; + $data->{diff_id} = $comp->{diff_id} unless defined $data->{diff_id}; + $data->{direction} = $comp->{direction} unless defined $data->{direction}; + + &my_die("zp value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{zp} and $comp->{zp} != $data->{zp}; + &my_die("zp_err value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{zp_err} and $comp->{zp_err} != $data->{zp_err}; + &my_die("astrom value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{astrom} and $comp->{astrom} != $data->{astrom}; + &my_die("exp_name value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{exp_name} and $comp->{exp_name} ne $data->{exp_name}; + &my_die("exp_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{exp_id} and $comp->{exp_id} != $data->{exp_id}; + &my_die("chip_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{chip_id} and $comp->{chip_id} != $data->{chip_id}; + &my_die("cam_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{cam_id} and $comp->{cam_id} != $data->{cam_id}; + &my_die("fake_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{fake_id} and $comp->{fake_id} != $data->{fake_id}; + &my_die("warp_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{warp_id} and $comp->{warp_id} != $data->{warp_id}; + &my_die("diff_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{diff_id} and $comp->{diff_id} != $data->{diff_id}; + &my_die("direction value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{direction} and $comp->{direction} != $data->{direction}; + + return 1; +} + +# Combine multiple files for MOPS +sub mops_combine +{ + my $data = shift; # Data + my $output = shift; # Output name + my $input = shift; # Input name + + $output = $ipprc->file_resolve( $output, 'create' ) or + &my_die( "Unable to resolve output file $output", $pub_id, $PS_EXIT_SYS_ERROR); + + my $command = "$ppMops $input $output"; + $command .= " -exp_name " . $data->{exp_name} if defined $data->{exp_name}; + $command .= " -exp_id " . $data->{exp_id} if defined $data->{exp_id}; + $command .= " -chip_id " . $data->{chip_id} if defined $data->{chip_id}; + $command .= " -cam_id " . $data->{cam_id} if defined $data->{cam_id}; + $command .= " -fake_id " . $data->{fake_id} if defined $data->{fake_id}; + $command .= " -warp_id " . $data->{warp_id} if defined $data->{warp_id}; + $command .= " -diff_id " . $data->{diff_id} if defined $data->{diff_id}; + $command .= " -inverse" if defined $data->{direction} and $data->{direction} == 0; + $command .= " -zp " . $data->{zp} if defined $data->{zp}; + $command .= " -zp_error " . $data->{zp_err} if defined $data->{zp_err}; + $command .= " -astrom_rms " . $data->{astrom} if defined $data->{astrom}; + + unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to translate", $pub_id, $PS_EXIT_SYS_ERROR) unless $success; + &my_die( "Unable to find translated file $output", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $output ); + } else { + print "Not running: $command\n"; + } + + return $output; +} @@ -210,4 +306,6 @@ $command .= " -pub_id $pub_id"; $command .= " -path_base $outroot"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; $command .= " -fault $fault"; $command .= " -dbname $dbname" if defined $dbname; Index: branches/eam_branches/20090820/ippScripts/scripts/rcserver_checkstatus.pl =================================================================== --- branches/eam_branches/20090820/ippScripts/scripts/rcserver_checkstatus.pl (revision 25206) +++ branches/eam_branches/20090820/ippScripts/scripts/rcserver_checkstatus.pl (revision 25766) @@ -40,10 +40,9 @@ # Parse the command-line arguments -my ($dest_id, $prod_id, $status_uri, $last_fileset, $limit); +my ($dest_id, $status_uri, $last_fileset, $limit); my ($dbname, $save_temps, $verbose, $no_update, $logfile); GetOptions( 'dest_id=s' => \$dest_id, # destination identifier - 'prod_id=s' => \$prod_id, # product identifier 'status_uri=s' => \$status_uri, # uri for the desination's status data store product 'last_fileset=s' => \$last_fileset, # name of last fileset seen on this datastore Index: branches/eam_branches/20090820/ippScripts/scripts/register_imfile.pl =================================================================== --- branches/eam_branches/20090820/ippScripts/scripts/register_imfile.pl (revision 25206) +++ branches/eam_branches/20090820/ippScripts/scripts/register_imfile.pl (revision 25766) @@ -33,6 +33,7 @@ my $ppStats = can_run( 'ppStats' ) or (warn "Can't find ppStats" and $missing_tools = 1); my $ppStatsFromMetadata = can_run( 'ppStatsFromMetadata' ) or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); - -my ($cache, $exp_id, $tmp_class_id, $tmp_exp_name, $uri, $dbname, $verbose, $no_update, $no_op, $logfile); +my $ppConfigDump = can_run( 'ppConfigDump' ) or (warn "Can't find ppConfigDump" and $missing_tools = 1); + +my ($cache, $exp_id, $tmp_class_id, $tmp_exp_name, $uri, $bytes, $md5sum, $dbname, $verbose, $no_update, $no_op, $logfile); GetOptions( 'caches' => \$cache, @@ -41,4 +42,6 @@ 'tmp_exp_name|n=s' => \$tmp_exp_name, 'uri|u=s' => \$uri, + 'bytes=s' => \$bytes, + 'md5sum=s' => \$md5sum, 'dbname|d=s' => \$dbname, # Database name 'verbose' => \$verbose, # Print to stdout @@ -106,4 +109,45 @@ chomp $out2; $cmdflags = $out2; + + # Manually parse the burntool_state entry. + my $burntoolState = 0; + my $isGPC1 = 0; + my $burntoolStateGood = 0; + foreach my $line (split /\n/, $out1) { + if ($line =~ /FPA.BURNTOOL.APPLIED/) { + $line =~ s/^\s+//; + $burntoolState = (split /\s+/, $line)[2]; + } + if ($line =~ /FPA.CAMERA/) { + $line =~ s/^\s+//; + if ((split /\s+/, $line)[2] eq 'GPC1') { + $isGPC1 = 1; + } + } + } + if ($isGPC1 != 1) { + $burntoolState = 0; # If it's not GPC1, you shouldn't have run burntool. + } + elsif (($isGPC1 == 1) && ($burntoolState == 1)) { + my $ppConfigDump_cmd = "$ppConfigDump -camera GPC1 -dump-camera -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + IPC::Cmd::run(command => $ppConfigDump_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform ppConfigDump"); + exit($error_code); + } + + # This is ugly, but doing a full parse for one entry is a bit wasteful. + foreach my $line (split /\n/, (join "", @$stdout_buf)) { + if ($line =~ /BURNTOOL.STATE.GOOD/) { + $line =~ s/^\s+//; + $burntoolStateGood = (split /\s+/, $line)[2]; + last; + } + } + $burntoolState = $burntoolStateGood; # Positive because this has the header table. + } + $cmdflags .= " -burntool_state $burntoolState "; } @@ -123,4 +167,6 @@ $command .= " -tmp_class_id $tmp_class_id"; # the original class_id supplied by the user, replace by ppStats CLASS.ID $command .= " -uri $uri "; +$command .= " -bytes $bytes" if $bytes; +$command .= " -md5sum $md5sum" if $md5sum and ($md5sum ne 'NULL'); $command .= " -hostname $host" if defined $host; $command .= " -dbname $dbname" if defined $dbname; Index: branches/eam_branches/20090820/ippScripts/scripts/stack_skycell.pl =================================================================== --- branches/eam_branches/20090820/ippScripts/scripts/stack_skycell.pl (revision 25206) +++ branches/eam_branches/20090820/ippScripts/scripts/stack_skycell.pl (revision 25766) @@ -65,4 +65,6 @@ my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $stack_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration +$| = 1; +print "I've set up: $stack_id\n"; # XXX camera is not known here; cannot use filerules... @@ -107,4 +109,6 @@ &my_die("Stack list contains less than two elements", $stack_id, $PS_EXIT_SYS_ERROR) unless scalar @$files >= 2; + +print "I've loaded my inputs: $stack_id\n"; # Parse the list of input files to get the tesselation, skycell identifiers and camera @@ -137,4 +141,6 @@ $ipprc->define_camera($camera); +print "I've configured everything: $stack_id\n"; + # Recipes to use based on reduction class $reduction = 'DEFAULT' unless defined $reduction; @@ -146,4 +152,19 @@ } +my $recipe; +{ + my $command = "$ppConfigDump -camera $camera -recipe PPSTACK $recipe_ppStack -dump-recipe PPSTACK -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $stack_id, $error_code); + } + $recipe = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $stack_id, $PS_EXIT_PROG_ERROR); +} +my $convolve = metadataLookupBool($recipe, 'CONVOLVE'); # Convolve inputs? + + # Generate MDC file with the inputs my $tess_base = basename($tess_id); @@ -160,5 +181,5 @@ my $mask = $ipprc->filename( "PSWARP.OUTPUT.MASK", $file->{path_base} ); # Mask name my $weight = $ipprc->filename( "PSWARP.OUTPUT.VARIANCE", $file->{path_base} ); # Weight name - my $psf = $ipprc->filename( "PSPHOT.PSF.SKY.SAVE", $file->{path_base} ); # PSF name + my $psf = $convolve ? $ipprc->filename( "PSPHOT.PSF.SKY.SAVE", $file->{path_base} ) : undef; # PSF name my $sources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $file->{path_base}); # Sources name @@ -166,5 +187,5 @@ &my_die("Mask $mask does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $mask ); &my_die("Weight $weight does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $weight ); - &my_die("PSF $psf does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $psf ); + &my_die("PSF $psf does not exist", $stack_id, $PS_EXIT_SYS_ERROR) if ($convolve and not $ipprc->file_exists( $psf )); &my_die("Sources $sources does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $sources ); @@ -172,12 +193,11 @@ print $listFile "\tMASK\tSTR\t" . $mask . "\n"; print $listFile "\tVARIANCE\tSTR\t" . $weight . "\n"; - print $listFile "\tPSF\tSTR\t" . $psf . "\n"; + print $listFile "\tPSF\tSTR\t" . $psf . "\n" if $convolve; print $listFile "\tSOURCES\tSTR\t" . $sources . "\n"; - ### XXX NEED TO UPDATE THESE appropriately - print $listFile "\tWEIGHTING\tF32\t" . 1.0 . "\n"; - print $listFile "END\n\n"; } + +print "I've checked everything: $stack_id\n"; # Get the output filenames @@ -200,4 +220,5 @@ my $cmdflags; + # Perform stacking @@ -266,4 +287,5 @@ } + print "I've stacked $listName: $stack_id\n"; } @@ -298,4 +320,6 @@ } +print "I've updated teh database: $stack_id\n"; + my $my_die_called = 0; @@ -340,20 +364,11 @@ print "Ensuring temporary images are deleted.\n"; - my $command = "$ppConfigDump -camera $camera -recipe PPSTACK $recipe_ppStack -dump-recipe PPSTACK -"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppConfigDump: $error_code", $stack_id, $error_code); - } - my $md = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $stack_id, $PS_EXIT_PROG_ERROR); - - my $temp_delete = metadataLookupBool($md, 'TEMP.DELETE'); # Delete temporary files? + + my $temp_delete = defined($recipe) and metadataLookupBool($recipe, 'TEMP.DELETE'); # Delete temp files? if ($temp_delete) { - my $temp_dir = metadataLookupStr($md, 'TEMP.DIR'); # Directory with temporary files - my $temp_image = metadataLookupStr($md, 'TEMP.IMAGE'); # Suffix for image - my $temp_mask = metadataLookupStr($md, 'TEMP.MASK'); # Suffix for mask - my $temp_weight = metadataLookupStr($md, 'TEMP.VARIANCE'); # Suffix for weight + my $temp_dir = metadataLookupStr($recipe, 'TEMP.DIR'); # Directory with temporary files + my $temp_image = metadataLookupStr($recipe, 'TEMP.IMAGE'); # Suffix for image + my $temp_mask = metadataLookupStr($recipe, 'TEMP.MASK'); # Suffix for mask + my $temp_weight = metadataLookupStr($recipe, 'TEMP.VARIANCE'); # Suffix for weight my $temp_root = basename($outroot); # Root name for temporary files @@ -372,4 +387,5 @@ } +print "I've reached the end of processing with a code of $?: $stack_id\n"; END { Index: branches/eam_branches/20090820/ippScripts/scripts/summit_copy.pl =================================================================== --- branches/eam_branches/20090820/ippScripts/scripts/summit_copy.pl (revision 25206) +++ branches/eam_branches/20090820/ippScripts/scripts/summit_copy.pl (revision 25766) @@ -12,4 +12,5 @@ use Sys::Hostname; +use Digest::MD5::File qw( file_md5_hex ); use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); use Pod::Usage qw( pod2usage ); @@ -21,5 +22,5 @@ # Parse the command-line arguments -my ( $uri, $filename, $compress, $bytes, $md5, $nebulous, $exp_name, $inst, $telescope, $class, $class_id, $end_stage, $workdir, +my ( $uri, $filename, $compress, $bytes, $md5, $nebulous, $exp_name, $inst, $telescope, $class, $class_id, $dbname, $verbose, $no_update, $no_op, $timeout, $copies ); GetOptions( @@ -35,6 +36,4 @@ 'class=s' => \$class, # Class level 'class_id=s' => \$class_id, # Class identifier - 'end_stage=s' => \$end_stage, # end of processing - 'workdir=s' => \$workdir, # workdir 'dbname=s' => \$dbname, # Database name 'verbose' => \$verbose, # Print to stdout @@ -62,8 +61,12 @@ my $pztool = can_run('pztool') or (warn "Can't find pztool" and $missing_tools = 1); +my $neblocate = can_run('neb-locate') + or (warn "Can't find neb-locate" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); exit($PS_EXIT_CONFIG_ERROR); } + +my $ipprc = PS::IPP::Config->new(); # dsget command @@ -97,4 +100,11 @@ } +# +# Find all instances of the new file and make sure they have the +# same checksum and size. +# if so pass the results to pztool to the results in pzDownloadImfile. +# uncomment this to turn on +my ($new_bytes, $new_md5) = check_instances($filename, $nebulous, $compress); + # command to update database $command = "$pztool -copydone"; @@ -106,8 +116,10 @@ $command .= " -class_id $class_id"; $command .= " -uri $filename"; -# $command .= " -workdir $workdir"; $command .= " -hostname $host"; -# $command .= " -end_stage $end_stage" if defined $end_stage; $command .= " -dbname $dbname" if defined $dbname; + +# XXX: TODO: see above. Don't do this until pztool and the DB have +# been updated +$command .= " -md5sum $new_md5 -bytes $new_bytes"; # update the database @@ -122,4 +134,86 @@ } else { print "skipping command: $command\n"; +} + +sub get_file_params +{ + my $filename = shift; + + my $size = -s $filename; + my $md5 = file_md5_hex($filename); + + return ($size, $md5); +} + +sub check_instances { + my $filename = shift; + my $nebulous = shift; + my $compress = shift; + + my @instances; + if ($nebulous) { + my $command = "$neblocate --path --all $filename"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) + = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + my_die("Unable to perform neb-locate: $error_code", + $exp_name, + $inst, + $telescope, + $class, + $class_id, + $uri, + $error_code + ); + } + @instances = split "\n", join "", @$stdout_buf; + } else { + if (!defined $ipprc) { + # we don't usually need this so defer instantaiating it + $ipprc = PS::IPP::Config->new(); + } + my $resolved = $ipprc->file_resolve($filename); + if ($resolved) { + $instances[0] = $resolved; + } + } + if (! scalar @instances) { + my_die("no instances", + $exp_name, + $inst, + $telescope, + $class, + $class_id, + $uri, + $PS_EXIT_UNKNOWN_ERROR + ); + } + + my ($new_bytes, $new_md5) = get_file_params($instances[0]); + + for (my $i = 1; $i < scalar @instances; $i++) { + my ($b, $m) = get_file_params($instances[$i]); + my $error = ""; + if ($b ne $new_bytes) { + $error = "size of $instances[$i] does not match $instances[0]"; + } elsif ($m ne $new_md5) { + $error = "md5sum of $instances[$i] does not match $instances[0]"; + } + if ($error) { + my_die($error, + $exp_name, + $inst, + $telescope, + $class, + $class_id, + $uri, + $PS_EXIT_DATA_ERROR + ); + } + } + + return ($new_bytes, $new_md5); + } Index: branches/eam_branches/20090820/ippScripts/scripts/warp_skycell.pl =================================================================== --- branches/eam_branches/20090820/ippScripts/scripts/warp_skycell.pl (revision 25206) +++ branches/eam_branches/20090820/ippScripts/scripts/warp_skycell.pl (revision 25766) @@ -183,4 +183,18 @@ close $astromFile; +# We need the recipe to determine if we care whether the PSF is generated or not +my $recipe; +{ + my $command = "$ppConfigDump -camera $camera -dump-recipe PSWARP -recipe PSWARP $recipe_pswarp -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR); + } + $recipe = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR); +} + # Run pswarp @@ -201,5 +215,4 @@ $command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID"; $command .= " -recipe PSWARP $recipe_pswarp"; - $command .= " -psf"; # Turn on PSF determination $command .= " -tracedest $traceDest -log $logDest"; $command .= " -threads $threads" if defined $threads; @@ -253,5 +266,5 @@ &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); - &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputPSF); + &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) if metadataLookupBool($recipe, 'PSF') and not $ipprc->file_exists($outputPSF); } Index: branches/eam_branches/20090820/ippTasks/Makefile.am =================================================================== --- branches/eam_branches/20090820/ippTasks/Makefile.am (revision 25206) +++ branches/eam_branches/20090820/ippTasks/Makefile.am (revision 25766) @@ -13,7 +13,9 @@ chip.pro \ camera.pro \ + addstar.pro \ fake.pro \ warp.pro \ magic.pro \ + destreak.pro \ diff.pro \ stack.pro \ @@ -25,5 +27,6 @@ pstamp.pro \ receive.pro \ - publish.pro + publish.pro \ + science.cleanup.pro other_files = \ Index: branches/eam_branches/20090820/ippTasks/addstar.pro =================================================================== --- branches/eam_branches/20090820/ippTasks/addstar.pro (revision 25766) +++ branches/eam_branches/20090820/ippTasks/addstar.pro (revision 25766) @@ -0,0 +1,233 @@ +## addstar.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the addstar analysis stage +## these tasks use the book addPendingExp + +# test for required global variables +check.globals + +book init addPendingExp + +macro addstar.status + book listbook addPendingExp +end + +macro addstar.reset + book init addPendingExp +end + +macro addstar.on + task addstar.exp.load + active true + end + task addstar.exp.run + active true + end + task addstar.revert + active true + end +end + +macro addstar.off + task addstar.exp.load + active false + end + task addstar.exp.run + active false + end + task addstar.revert + active false + end +end + +# this variable will cycle through the known database names +$addstar_DB = 0 +$addstar_revert_DB = 0 + +# select images ready for addstar analysis +# new entries are added to addPendingExp +# skip already-present entries +task addstar.exp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/addstar.exp.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -pendingexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_DB + $run = $run -dbname $DB:$addstar_DB + $addstar_DB ++ + if ($addstar_DB >= $DB:n) set addstar_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout addPendingExp -key add_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook addPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup addPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the addstar script on pending exposures +task addstar.exp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10 + + task.exec + book npages addPendingExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in addPendingExp (pantaskState == INIT) + book getpage addPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword addPendingExp $pageName pantaskState RUN + book getword addPendingExp $pageName camera -var CAMERA + book getword addPendingExp $pageName exp_tag -var EXP_TAG + book getword addPendingExp $pageName add_id -var ADD_ID + book getword addPendingExp $pageName cam_id -var CAM_ID + book getword addPendingExp $pageName workdir -var WORKDIR_TEMPLATE + book getword addPendingExp $pageName dvodb -var DVODB + book getword addPendingExp $pageName dbname -var DBNAME + book getword addPendingExp $pageName reduction -var REDUCTION + book getword addPendingExp $pageName state -var RUN_STATE + + # specify choice of remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s/%s.add.%s" $WORKDIR $EXP_TAG $EXP_TAG $ADD_ID + sprintf camroot "%s/%s/%s.cm.%s" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID + + stdout $LOGDIR/addstar.exp.log + stderr $LOGDIR/addstar.exp.log + + $run = addstar_run.pl --exp_tag $EXP_TAG --add_id $ADD_ID --camera $CAMERA --outroot $outroot --camroot $camroot --redirect-output --run-state $RUN_STATE + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + if ("$DVODB" != "NULL") + $run = $run --dvodb $DVODB + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit addPendingExp $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword addPendingExp $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword addPendingExp $options:0 pantaskState TIMEOUT + end +end + +task addstar.revert + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB + $run = $run -dbname $DB:$addstar_revert_DB + $addstar_revert_DB ++ + if ($addstar_revert_DB >= $DB:n) set addstar_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: branches/eam_branches/20090820/ippTasks/camera.pro =================================================================== --- branches/eam_branches/20090820/ippTasks/camera.pro (revision 25206) +++ branches/eam_branches/20090820/ippTasks/camera.pro (revision 25766) @@ -23,4 +23,7 @@ active true end + task camera.revert + active true + end end @@ -30,4 +33,7 @@ end task camera.exp.run + active false + end + task camera.revert active false end Index: branches/eam_branches/20090820/ippTasks/chip.pro =================================================================== --- branches/eam_branches/20090820/ippTasks/chip.pro (revision 25206) +++ branches/eam_branches/20090820/ippTasks/chip.pro (revision 25766) @@ -26,4 +26,7 @@ active true end + task chip.revert + active true + end end @@ -36,4 +39,7 @@ end task chip.advanceexp + active false + end + task chip.revert active false end Index: branches/eam_branches/20090820/ippTasks/destreak.pro =================================================================== --- branches/eam_branches/20090820/ippTasks/destreak.pro (revision 25766) +++ branches/eam_branches/20090820/ippTasks/destreak.pro (revision 25766) @@ -0,0 +1,490 @@ +## destreak.pro : support for the streak removal : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/destreak +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init magicToDS +book init magicDSToRevert + +### indexes into Database lists +$magicToDS_DB = 0 +$magicDSAdvance_DB = 0 +$magicDSToRevert_DB = 0 +$magicDSCompletedRevert_DB = 0 + +#list of stages +$DS_STAGE:n = 0 +list DS_STAGE -add "raw" +list DS_STAGE -add "chip" +list DS_STAGE -add "camera" +list DS_STAGE -add "warp" +list DS_STAGE -add "diff" + +$magicDSStage = 0 +$magicDSRevertStage = 0 + +### Check status of tasks +macro magic.status + book listbook magicToDS + book listbook magicDSToRevert +end + +### Reset tasks +macro magic.reset + book init magicToDS + book init magicDSToRevert +end + +### Turn tasks on +macro destreak.on + # destreak and revert should not run at the same time + destreak.revert.off + + task destreak.load + active true + end + task destreak.run + active true + end + task destreak.advance + active true + end +end + +macro destreak.revert.on + # destreak and revert should not run at the same time + destreak.off + + task destreak.revert.load + active true + end + task destreak.revert.run + active true + end + task destreak.completed.revert + active true + end +end + +### Turn tasks off +macro destreak.off + task destreak.load + active false + end + task destreak.run + active false + end + task destreak.advance + active false + end +end + +macro destreak.revert.off + task destreak.revert.load + active false + end + task destreak.revert.run + active false + end + task destreak.completed.revert + active false + end +end + +macro destreak.status + echo magicToDS + book listbook magicToDS + echo "" + echo magicDSToRevert + book listbook magicDSToRevert +end + +task destreak.load + host local + + periods -poll $LOADPOLL + # this query can take a long time (XXX: the long time should be fixed now) + periods -exec 10 + periods -timeout 120 + npending 1 + +# stdout NULL +# stderr $LOGSUBDIR/destreak.load.log + + task.exec + $run = magicdstool -todestreak -limit 120 -stage $DS_STAGE:$magicDSStage + $magicDSStage ++ + if ($magicDSStage >= $DS_STAGE:n) set magicDSStage = 0 + + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$magicToDS_DB + $run = $run -dbname $DB:$magicToDS_DB + + # only bump database number after we have gone through all of the stages + if ($magicDSStage == 0) + $magicToDS_DB ++ + if ($magicToDS_DB >= $DB:n) set magicToDS_DB = 0 + end + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicToDS -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicToDS + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicToDS + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task destreak.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + stdout $LOGSUBDIR/destreak.run.log + stderr $LOGSUBDIR/destreak.run.log + book npages magicToDS -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicToDS 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicToDS $pageName pantaskState RUN + book getword magicToDS $pageName exp_id -var EXP_ID + book getword magicToDS $pageName magic_ds_id -var MAGIC_DS_ID + book getword magicToDS $pageName camera -var CAMERA + book getword magicToDS $pageName streaks_uri -var STREAKS + book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS + book getword magicToDS $pageName stage -var STAGE + book getword magicToDS $pageName stage_id -var STAGE_ID + book getword magicToDS $pageName component -var COMPONENT + book getword magicToDS $pageName uri -var URI + book getword magicToDS $pageName path_base -var PATH_BASE + book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE + book getword magicToDS $pageName outroot -var OUTROOT + book getword magicToDS $pageName recoveryroot -var RECROOT + book getword magicToDS $pageName re_place -var REPLACE + book getword magicToDS $pageName dbname -var DBNAME + + sprintf logfile "%s/%s.mds.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT + + substr $COMPONENT 0 3 COMP_HEAD + if ("$COMP_HEAD" == "sky") + set.host.for.skycell $COMPONENT + else + # assume component is a class_id, if not we will default to anyhost + set.host.for.camera $CAMERA $COMPONENT + end + + # TODO: do not add recoveryroot or replace if they are null or zero + + $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit 0 + process_exit magicToDS $options:0 $JOB_STATUS + end + + # locked list + task.exit default + showcommand failure + process_exit magicToDS $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword magicToDS $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicToDS $options:0 pantaskState TIMEOUT + end +end + +task destreak.advance + # task to finish processing for magicDSRuns + host local + + periods -poll $LOADPOLL + #periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 20 + npending 1 + +# stdout NULL +# stderr $LOGSUBDIR/destreak.advance.log + + task.exec + $run = magicdstool -advancerun + if ($DB:n != 0) + + $run = $run -dbname $DB:$magicDSAdvance_DB + + $magicDSAdvance_DB ++ + if ($magicDSAdvance_DB >= $DB:n) set magicDSAdvance_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task destreak.revert.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 20 + npending 1 + active false + + stdout NULL + stderr $LOGSUBDIR/destreak.revert.log + + task.exec + $run = magicdstool -torevert -stage $DS_STAGE:$magicDSRevertStage + $magicDSRevertStage ++ + if ($magicDSRevertStage >= $DS_STAGE:n) set magicDSRevertStage = 0 + + if ($DB:n == 0) + option DEFAULT + else + + # save the DB name for the exit tasks + option $DB:$magicDSToRevert_DB + $run = $run -dbname $DB:$magicDSToRevert_DB + + # only bump database number after we have gone through all of the stages + if ($magicDSRevertStage == 0) + $magicDSToRevert_DB ++ + if ($magicDSToRevert_DB >= $DB:n) set magicDSToRevert_DB = 0 + end + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicDSToRevert -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicDSToRevert + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicDSToRevert + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task destreak.revert.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + stdout $LOGSUBDIR/destreak.revert.run.log + stderr $LOGSUBDIR/destreak.revert.run.log + + book npages magicDSToRevert -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicDSToRevert 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicDSToRevert $pageName pantaskState RUN + book getword magicDSToRevert $pageName exp_id -var EXP_ID + book getword magicDSToRevert $pageName magic_ds_id -var MAGIC_DS_ID + book getword magicDSToRevert $pageName camera -var CAMERA + book getword magicDSToRevert $pageName stage -var STAGE + book getword magicDSToRevert $pageName stage_id -var STAGE_ID + book getword magicDSToRevert $pageName component -var COMPONENT + book getword magicDSToRevert $pageName path_base -var PATH_BASE + book getword magicDSToRevert $pageName cam_path_base -var CAM_PATH_BASE + book getword magicDSToRevert $pageName outroot -var OUTROOT + book getword magciDSToRevert $pageName bytes -var BYTES + book getword magciDSToRevert $pageName md5sum -var md5sum +# book getword magicDSToRevert $pageName recoveryroot -var RECROOT + book getword magicDSToRevert $pageName re_place -var REPLACE + book getword magicDSToRevert $pageName dbname -var DBNAME + + sprintf logfile "%s/%s.mds.revert.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT + + substr $COMPONENT 0 3 COMP_HEAD + if ("$COMP_HEAD" == "sky") + set.host.for.skycell $COMPONENT + else + # assume component is a class_id, if not we will default to anyhost + set.host.for.camera $CAMERA $COMPONENT + end + + $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --replace $REPLACE + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit 0 + process_exit magicDSToRevert $options:0 $JOB_STATUS + end + + # locked list + task.exit default + showcommand failure + process_exit magicDSToRevert $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword magicDSToRevert $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicDSToRevert $options:0 pantaskState TIMEOUT + end +end + +task destreak.completed.revert + # task to finish processing for magicDSRuns being reverted or restored + host local + + periods -poll $LOADPOLL + #periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 20 + npending 1 + active false + + stdout NULL + stderr $LOGSUBDIR/destreak.completed.revert.log + + task.exec + $run = magicdstool -completedrevert + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$magicDSCompletedRevert_DB + + $run = $run -dbname $DB:$magicDSCompletedRevert_DB + + $magicDSCompletedRevert_DB ++ + if ($magicDSCompletedRevert_DB >= $DB:n) set magicDSCompletedRevert_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: branches/eam_branches/20090820/ippTasks/detrend.cleanup.pro =================================================================== --- branches/eam_branches/20090820/ippTasks/detrend.cleanup.pro (revision 25206) +++ branches/eam_branches/20090820/ippTasks/detrend.cleanup.pro (revision 25766) @@ -234,5 +234,5 @@ book getword detCleanupProcessedImfile $pageName class_id -var CLASS_ID book getword detCleanupProcessedImfile $pageName camera -var CAMERA - book getword detCleanupProcessedImfile $pageName state -var CLEANUP_MODE + book getword detCleanupProcessedImfile $pageName data_state -var CLEANUP_MODE book getword detCleanupProcessedImfile $pageName dbname -var DBNAME @@ -244,5 +244,5 @@ # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.process.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage detrend.process.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE add_standard_args run @@ -352,5 +352,5 @@ book getword detCleanupProcessedExp $pageName exp_id -var EXP_ID book getword detCleanupProcessedExp $pageName camera -var CAMERA - book getword detCleanupProcessedExp $pageName state -var CLEANUP_MODE + book getword detCleanupProcessedExp $pageName data_state -var CLEANUP_MODE book getword detCleanupProcessedExp $pageName dbname -var DBNAME @@ -362,5 +362,5 @@ # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.process.exp --stage_id $DET_ID --exp_id $EXP_ID --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage detrend.process.exp --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE add_standard_args run @@ -470,5 +470,5 @@ book getword detCleanupStackedImfile $pageName class_id -var CLASS_ID book getword detCleanupStackedImfile $pageName camera -var CAMERA - book getword detCleanupStackedImfile $pageName state -var CLEANUP_MODE + book getword detCleanupStackedImfile $pageName data_state -var CLEANUP_MODE book getword detCleanupStackedImfile $pageName dbname -var DBNAME @@ -480,5 +480,5 @@ # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.stack.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage detrend.stack.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE add_standard_args run @@ -588,5 +588,5 @@ book getword detCleanupNormStatImfile $pageName iteration -var ITERATION book getword detCleanupNormStatImfile $pageName camera -var CAMERA - book getword detCleanupNormStatImfile $pageName state -var CLEANUP_MODE + book getword detCleanupNormStatImfile $pageName data_state -var CLEANUP_MODE book getword detCleanupNormStatImfile $pageName dbname -var DBNAME @@ -598,5 +598,5 @@ # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.normstat.imfile --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage detrend.normstat.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE add_standard_args run @@ -704,5 +704,5 @@ book getword detCleanupNormImfile $pageName class_id -var CLASS_ID book getword detCleanupNormImfile $pageName camera -var CAMERA - book getword detCleanupNormImfile $pageName state -var CLEANUP_MODE + book getword detCleanupNormImfile $pageName data_state -var CLEANUP_MODE book getword detCleanupNormImfile $pageName dbname -var DBNAME @@ -714,5 +714,5 @@ # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.norm.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage detrend.norm.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE add_standard_args run @@ -819,5 +819,5 @@ book getword detCleanupNormExp $pageName iteration -var ITERATION book getword detCleanupNormExp $pageName camera -var CAMERA - book getword detCleanupNormExp $pageName state -var CLEANUP_MODE + book getword detCleanupNormExp $pageName data_state -var CLEANUP_MODE book getword detCleanupNormExp $pageName dbname -var DBNAME @@ -829,5 +829,5 @@ # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.norm.exp --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage detrend.norm.exp --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE add_standard_args run @@ -937,5 +937,5 @@ book getword detCleanupResidImfile $pageName iteration -var ITERATION book getword detCleanupResidImfile $pageName camera -var CAMERA - book getword detCleanupResidImfile $pageName state -var CLEANUP_MODE + book getword detCleanupResidImfile $pageName data_state -var CLEANUP_MODE book getword detCleanupResidImfile $pageName dbname -var DBNAME @@ -947,5 +947,5 @@ # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.resid.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage detrend.resid.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE add_standard_args run @@ -1056,5 +1056,5 @@ book getword detCleanupResidExp $pageName iteration -var ITERATION book getword detCleanupResidExp $pageName camera -var CAMERA - book getword detCleanupResidExp $pageName state -var CLEANUP_MODE + book getword detCleanupResidExp $pageName data_state -var CLEANUP_MODE book getword detCleanupResidExp $pageName dbname -var DBNAME @@ -1066,5 +1066,5 @@ # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.resid.exp --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage detrend.resid.exp --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE add_standard_args run Index: branches/eam_branches/20090820/ippTasks/diff.pro =================================================================== --- branches/eam_branches/20090820/ippTasks/diff.pro (revision 25206) +++ branches/eam_branches/20090820/ippTasks/diff.pro (revision 25766) @@ -14,4 +14,5 @@ ### Database lists $diffSkycell_DB = 0 +$diffAdvance_DB = 0 #$diffCleanup_DB = 0 @@ -36,4 +37,7 @@ active true end + task diff.advance + active true + end end @@ -44,4 +48,7 @@ end task diff.skycell.run + active false + end + task diff.advance active false end @@ -198,4 +205,56 @@ end end + + +# Advance exposures which have completed +task diff.advance + host local + + periods -poll $LOADPOLL +# periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGDIR/diff.advance.log + + task.exec + if ($LABEL:n == 0) break + $run = difftool -advance + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$diffAdvance_DB + $run = $run -dbname $DB:$diffAdvance_DB + $diffAdvance_DB ++ + if ($diffAdvance_DB >= $DB:n) set diffAdvance_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + # # select images ready for diff analysis Index: branches/eam_branches/20090820/ippTasks/dist.pro =================================================================== --- branches/eam_branches/20090820/ippTasks/dist.pro (revision 25206) +++ branches/eam_branches/20090820/ippTasks/dist.pro (revision 25766) @@ -16,4 +16,17 @@ $distQueue_DB = 0 +### list of stages +#list of stages +$DIST_STAGE:n = 0 +list DIST_STAGE -add "raw" +list DIST_STAGE -add "chip" +list DIST_STAGE -add "camera" +list DIST_STAGE -add "fake" +list DIST_STAGE -add "warp" +list DIST_STAGE -add "diff" +list DIST_STAGE -add "stack" + +$currentStage = 0 + ### Check status of tasks macro dist.status @@ -42,10 +55,6 @@ active true end - task dist.queueruns -# We aren't ready to run this task yet. It will queue much too much -# active true - active false - end -end +end + macro dist.off task dist.process.load @@ -59,7 +68,4 @@ end task dist.advance.run - active false - end - task dist.queueruns active false end @@ -74,9 +80,13 @@ npending 1 - stdout NULL - stderr $LOGDIR/dist.process.log task.exec - $run = disttool -pendingcomponent + # stdout NULL + # stderr $LOGSUBDIR/dist.process.load.log + + $run = disttool -pendingcomponent -stage $DIST_STAGE:$currentStage + $currentStage ++ + if ($currentStage >= $DIST_STAGE:n) set currentStage = 0 + if ($DB:n == 0) option DEFAULT @@ -85,6 +95,11 @@ option $DB:$distToProcess_DB $run = $run -dbname $DB:$distToProcess_DB - $distToProcess_DB ++ - if ($distToProcess_DB >= $DB:n) set distToProcess_DB = 0 + + # only increment the database number after we have gone through all of + # the stages + if ($currentStage == 0) + $distToProcess_DB ++ + if ($distToProcess_DB >= $DB:n) set distToProcess_DB = 0 + end end add_poll_args run @@ -134,9 +149,11 @@ if ("$pageName" == "NULL") break +# echo running $pageName + book setword distToProcess $pageName pantaskState RUN book getword distToProcess $pageName dist_id -var DIST_ID book getword distToProcess $pageName camera -var CAMERA - book getword distToProcess $pageName stage -var STAGE - book getword distToProcess $pageName stage_id -var STAGE_ID + book getword distToProcess $pageName stage -var DIST_STAGE + book getword distToProcess $pageName stage_id -var DIST_STAGE_ID book getword distToProcess $pageName clean -var CLEAN book getword distToProcess $pageName component -var COMPONENT @@ -159,6 +176,6 @@ $EXTRA_ARGS = $EXTRA_ARGS --magicked end - # no_magic is output as integer due to the union in the sql - if ($NO_MAGIC) + # is this right for stack and fake? + if ("$NO_MAGIC" == "T") $EXTRA_ARGS = $EXTRA_ARGS --no_magic end @@ -170,5 +187,4 @@ # set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR # host anyhost - substr $COMPONENT 0 3 COMP_HEAD @@ -180,10 +196,9 @@ end - sprintf logfile "%s/dist.%s.%s.log" $OUTDIR $DIST_ID $COMPONENT - stdout $logfile - stderr $logfile - - $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE $EXTRA_ARGS --outdir $OUTDIR --logfile $logfile +# stdout $logfile +# stderr $logfile + + $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $DIST_STAGE --stage_id $DIST_STAGE_ID --component $COMPONENT --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE $EXTRA_ARGS --outdir $OUTDIR --logfile $logfile add_standard_args run @@ -211,5 +226,4 @@ end - task dist.advance.load host local @@ -220,8 +234,7 @@ npending 1 - stdout NULL - stderr $LOGDIR/dist.advance.load.log - task.exec + # stderr $LOGSUBDIR/dist.advance.load.log + $run = disttool -toadvance if ($DB:n == 0) @@ -261,4 +274,5 @@ end end + task dist.advance.run @@ -291,6 +305,6 @@ sprintf logfile "%s/dist.advance.%s.log" $OUTDIR $DIST_ID - stdout $logfile - stderr $logfile +# stdout $logfile +# stderr $logfile $run = dist_advancerun.pl --dist_id $DIST_ID --stage $STAGE --stage_id $STAGE_ID --outdir $OUTDIR $EXTRA_ARGS --logfile $logfile @@ -318,64 +332,2 @@ end end - -task dist.queueruns -# host local - - periods -poll $RUNPOLL - periods -exec 30 - periods -timeout 45 - npending 1 - - active false - -# stdout $LOGDIR/dist.queuruns -# stderr $LOGDIR/dist.queueruns - - task.exec - $MYARGS = "" - # assume that we need magic unless we are running simtest - if ($?SIMTEST_CAMERA != 0) - $MYARGS = --no_magic - end - - $run = dist_queue_runs.pl $MYARGS --stage_limit 16 --logfile $LOGDIR/dist.queueruns - - if ($DB:n == 0) - $DBNAME = DEFAULT - else - # save the DB name for add_standard_args - $DBNAME = $DB:$distQueue_DB - $distQueue_DB ++ - if ($distQueue_DB >= $DB:n) set distQueue_DB = 0 - end - - host anyhost - - add_standard_args run - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - task.exit $EXIT_SUCCESS - # nothing to do - end - - # default exit status - task.exit default - showcommand failure - end - - # operation timed out? - task.exit timeout - showcommand timeout - end - - # operation timed out? - task.exit crash - showcommand crash - end -end Index: branches/eam_branches/20090820/ippTasks/fake.pro =================================================================== --- branches/eam_branches/20090820/ippTasks/fake.pro (revision 25206) +++ branches/eam_branches/20090820/ippTasks/fake.pro (revision 25766) @@ -26,4 +26,7 @@ active true end + task fake.revert + active true + end end @@ -36,4 +39,7 @@ end task fake.advanceexp + active false + end + task fake.revert active false end Index: branches/eam_branches/20090820/ippTasks/ipphosts.mhpcc.config =================================================================== --- branches/eam_branches/20090820/ippTasks/ipphosts.mhpcc.config (revision 25206) +++ branches/eam_branches/20090820/ippTasks/ipphosts.mhpcc.config (revision 25766) @@ -3,38 +3,51 @@ ipphosts METADATA camera STR skycell - count S32 12 - sky00 STR ipp024 - sky01 STR ipp023 - sky02 STR ipp026 - sky03 STR ipp028 - sky04 STR ipp029 - sky05 STR ipp030 - sky06 STR ipp031 - sky07 STR ipp032 - sky08 STR ipp033 - sky09 STR ipp034 - sky10 STR ipp035 - sky11 STR ipp036 -END - -ipphosts METADATA - camera STR MEGACAM - ccd00 STR po00 - ccd01 STR po01 - ccd02 STR po02 - ccd03 STR po03 - ccd04 STR po04 - ccd05 STR po05 - ccd06 STR po06 - ccd07 STR po07 - ccd08 STR po08 - ccd09 STR po09 -END - -ipphosts METADATA - camera STR CFH12K - ccd00 STR po10 - ccd01 STR po11 - ccd02 STR po12 + count S32 46 + sky00 STR ipp006 + sky01 STR ipp007 + sky02 STR ipp008 + sky03 STR ipp009 + sky04 STR ipp010 + sky05 STR ipp011 + sky06 STR ipp012 + sky07 STR ipp013 + sky08 STR ipp014 + sky09 STR ipp015 + sky10 STR ipp016 + sky11 STR ipp017 + sky12 STR ipp018 + sky13 STR ipp019 + sky14 STR ipp020 + sky15 STR ipp021 + sky16 STR ipp023 + sky17 STR ipp024 + sky18 STR ipp025 + sky19 STR ipp026 + sky20 STR ipp027 + sky21 STR ipp028 + sky22 STR ipp029 + sky23 STR ipp030 + sky24 STR ipp031 + sky25 STR ipp032 + sky26 STR ipp033 + sky27 STR ipp034 + sky28 STR ipp035 + sky29 STR ipp036 + sky30 STR ipp038 + sky31 STR ipp039 + sky32 STR ipp040 + sky33 STR ipp041 + sky34 STR ipp042 + sky35 STR ipp043 + sky36 STR ipp044 + sky37 STR ipp045 + sky38 STR ipp046 + sky39 STR ipp047 + sky40 STR ipp048 + sky41 STR ipp049 + sky42 STR ipp050 + sky43 STR ipp051 + sky44 STR ipp052 + sky45 STR ipp053 END @@ -114,6 +127,6 @@ XY73 STR ipp015 XY74 STR ipp015 - XY75 STR ipp053 - XY76 STR ipp053 + XY75 STR ipp025 + XY76 STR ipp025 END @@ -193,5 +206,5 @@ ota73 STR ipp015 ota74 STR ipp015 - ota75 STR ipp053 - ota76 STR ipp053 + ota75 STR ipp025 + ota76 STR ipp025 END Index: branches/eam_branches/20090820/ippTasks/magic.pro =================================================================== --- branches/eam_branches/20090820/ippTasks/magic.pro (revision 25206) +++ branches/eam_branches/20090820/ippTasks/magic.pro (revision 25766) @@ -10,21 +10,8 @@ book init magicToTree book init magicToProcess -book init magicToDS -book init magicDSToRevert ### Database lists $magicToTree_DB = 0 $magicToProcess_DB = 0 -$magicToDS_DB = 0 -$magicDSToRevert_DB = 0 - -#list of stages -$STAGE:n = 0 -list STAGE -add "raw" -list STAGE -add "chip" -list STAGE -add "camera" -list STAGE -add "warp" -list STAGE -add "diff" -$magicDSRevertStage = 0 ### Check status of tasks @@ -32,6 +19,4 @@ book listbook magicToTree book listbook magicToProcess - book listbook magicToDS - book listbook magicDSToRevert end @@ -40,6 +25,4 @@ book init magicToTree book init magicToProcess - book init magicToDS - book init magicDSToRevert end @@ -58,20 +41,6 @@ active true end - task magic.destreak.load - active true - end - task magic.destreak.run - active true - end -end - -macro magic.ds.revert.on - task magic.ds.revert.load - active true - end - task magic.ds.revert.run - active true - end -end +end + ### Turn tasks off macro magic.off @@ -88,46 +57,5 @@ active false end - task magic.destreak.load - active false - end - task magic.destreak.run - active false - end -end - -macro magic.ds.revert.off - task magic.ds.revert.load - active false - end - task magic.ds.revert.run - active false - end -end -macro magic.ds.off - task magic.destreak.load - active false - end - task magic.destreak.run - active false - end -end -macro magic.ds.on - task magic.destreak.load - active true - end - task magic.destreak.run - active true - end -end - -macro magic.ds.status - echo magicToDS - book listbook magicToDS - echo "" - echo magicDSToRevert - book listbook magicDSToRevert -end - - +end task magic.tree.load @@ -400,279 +328,2 @@ end end - -task magic.destreak.load - host local - - periods -poll $LOADPOLL - # this query can take a long time - periods -exec 10 - periods -timeout 120 - npending 1 - - stdout NULL - stderr $LOGDIR/magic.destreak.log - - task.exec - $run = magicdstool -todestreak -limit 120 - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$magicToDS_DB - $run = $run -dbname $DB:$magicToDS_DB - $magicToDS_DB ++ - if ($magicToDS_DB >= $DB:n) set magicToDS_DB = 0 - end - add_poll_args run - add_poll_labels run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout magicToDS -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook magicToDS - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup magicToDS - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -task magic.destreak.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - - task.exec - book npages magicToDS -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images (pantaskState == INIT) - book getpage magicToDS 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword magicToDS $pageName pantaskState RUN - book getword magicToDS $pageName exp_id -var EXP_ID - book getword magicToDS $pageName magic_ds_id -var MAGIC_DS_ID - book getword magicToDS $pageName camera -var CAMERA - book getword magicToDS $pageName streaks_uri -var STREAKS - book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS - book getword magicToDS $pageName stage -var STAGE - book getword magicToDS $pageName stage_id -var STAGE_ID - book getword magicToDS $pageName component -var COMPONENT - book getword magicToDS $pageName uri -var URI - book getword magicToDS $pageName path_base -var PATH_BASE - book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE - book getword magicToDS $pageName outroot -var OUTROOT - book getword magicToDS $pageName recoveryroot -var RECROOT - book getword magicToDS $pageName re_place -var REPLACE - book getword magicToDS $pageName dbname -var DBNAME - - sprintf logfile "%s/%s.mds.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT - - substr $COMPONENT 0 3 COMP_HEAD - if ("$COMP_HEAD" == "sky") - set.host.for.skycell $COMPONENT - else - # assume component is a class_id, if not we will default to anyhost - set.host.for.camera $CAMERA $COMPONENT - end - - # TODO: do not add recoveryroot or replace if they are null or zero - - $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE - - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit 0 - process_exit magicToDS $options:0 $JOB_STATUS - end - - # locked list - task.exit default - showcommand failure - process_exit magicToDS $options:0 $JOB_STATUS - end - - task.exit crash - showcommand crash - book setword magicToDS $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword magicToDS $options:0 pantaskState TIMEOUT - end -end - -task magic.ds.revert.load - host local - - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 20 - npending 1 - active false - - stdout NULL - stderr $LOGDIR/magic.ds.revert.log - - task.exec - $run = magicdstool -torevert -stage $STAGE:$magicDSRevertStage - $magicDSRevertStage ++ - if ($magicDSRevertStage >= $STAGE:n) set magicDSRevertStage = 0 - - if ($DB:n == 0) - option DEFAULT - else - - # save the DB name for the exit tasks - option $DB:$magicDSToRevert_DB - $run = $run -dbname $DB:$magicDSToRevert_DB - - # only bump database number after we have gone through all of the stages - if ($magicDSRevertStage == 0) - $magicDSToRevert_DB ++ - end - if ($magicDSToRevert_DB >= $DB:n) set magicDSToRevert_DB = 0 - end - add_poll_args run - add_poll_labels run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout magicDSToRevert -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook magicDSToRevert - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup magicDSToRevert - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -task magic.ds.revert.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - active false - - task.exec - book npages magicDSToRevert -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images (pantaskState == INIT) - book getpage magicDSToRevert 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword magicDSToRevert $pageName pantaskState RUN - book getword magicDSToRevert $pageName exp_id -var EXP_ID - book getword magicDSToRevert $pageName magic_ds_id -var MAGIC_DS_ID - book getword magicDSToRevert $pageName camera -var CAMERA - book getword magicDSToRevert $pageName stage -var STAGE - book getword magicDSToRevert $pageName stage_id -var STAGE_ID - book getword magicDSToRevert $pageName component -var COMPONENT - book getword magicDSToRevert $pageName path_base -var PATH_BASE - book getword magicDSToRevert $pageName cam_path_base -var CAM_PATH_BASE - book getword magicDSToRevert $pageName outroot -var OUTROOT - book getword magciDSToRevert $pageName bytes -var BYTES - book getword magciDSToRevert $pageName md5sum -var md5sum -# book getword magicDSToRevert $pageName recoveryroot -var RECROOT - book getword magicDSToRevert $pageName re_place -var REPLACE - book getword magicDSToRevert $pageName dbname -var DBNAME - - sprintf logfile "%s/%s.mds.revert.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT - - substr $COMPONENT 0 3 COMP_HEAD - if ("$COMP_HEAD" == "sky") - set.host.for.skycell $COMPONENT - else - # assume component is a class_id, if not we will default to anyhost - set.host.for.camera $CAMERA $COMPONENT - end - - $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --replace $REPLACE - - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit 0 - process_exit magicDSToRevert $options:0 $JOB_STATUS - end - - # locked list - task.exit default - showcommand failure - process_exit magicDSToRevert $options:0 $JOB_STATUS - end - - task.exit crash - showcommand crash - book setword magicDSToRevert $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword magicDSToRevert $options:0 pantaskState TIMEOUT - end -end - Index: branches/eam_branches/20090820/ippTasks/pantasks.pro =================================================================== --- branches/eam_branches/20090820/ippTasks/pantasks.pro (revision 25206) +++ branches/eam_branches/20090820/ippTasks/pantasks.pro (revision 25766) @@ -1,16 +1,20 @@ ## pantasks.pro : globals and support macros : -*- sh -*- +# globals that may be modified by the user -- only init if not set +if ($?NEBULOUS == 0) set NEBULOUS = 0 +if ($?NETWORK == 0) set NETWORK = 1 +if ($?PARALLEL == 0) set PARALLEL = 1 +if ($?VERBOSE == 0) set VERBOSE = 1 +if ($?LABEL:n == 0) set LABEL:n = 0 +if ($?POLLLIMIT == 0) set POLLLIMIT = 32 +if ($?KEEP_FAILURES == 0) set KEEP_FAILURES = 0 + +if ($?LOGDIR == 0) + $LOGDIR = `pwd` + $LOGDIR = $LOGDIR/pantasks_logs + mkdir $LOGDIR +end + # globals used to control system behavior : these should be in uppercase -$NEBULOUS = 0 -$NETWORK = 1 -$PARALLEL = 1 -$VERBOSE = 1 -$LABEL:n = 0 -$POLLLIMIT = 32 -$LOGDIR = `pwd` -$LOGDIR = $LOGDIR/pantasks_logs -mkdir $LOGDIR -$KEEP_FAILURES = 0 - $LOADPOLL = 1.0 $LOADEXEC = 5.0 @@ -32,5 +36,5 @@ # very basic values: set these with init.copy.mhpcc -$default_host = anyhost +$default_host = any $workdir_template = `pwd` @@ -147,4 +151,42 @@ end +macro save.labels + if ($0 != 1) + echo "USAGE: save.labels" + break + end + if ($?LABEL:n == 0) + echo "no labels defined" + end + if ($LABEL:n == 0) + echo "no labels defined" + end + + exec rm -f labels.list + local i + for i 0 $LABEL:n + exec echo $LABEL:$i >> labels.list + end +end + +macro load.labels + if ($0 != 1) + echo "USAGE: load.labels" + break + end + + file labels.list found + if (not($found)) + echo "no saved labels in labels.list" + return + end + + list mylabels -x "cat labels.list" + local i + for i 0 $mylabels:n + add.label $mylabels:$i + end +end + macro init.isp list DB -add isp @@ -213,4 +255,5 @@ chip.on camera.on + addstar.on fake.on warp.on @@ -225,4 +268,5 @@ chip.off camera.off + addstar.off fake.off warp.off @@ -242,4 +286,5 @@ module chip.pro module camera.pro + module addstar.pro module fake.pro module warp.pro Index: branches/eam_branches/20090820/ippTasks/pstamp.pro =================================================================== --- branches/eam_branches/20090820/ippTasks/pstamp.pro (revision 25206) +++ branches/eam_branches/20090820/ippTasks/pstamp.pro (revision 25766) @@ -1,4 +1,8 @@ - -#$VERBOSE = 3 +# pstamp.pro : Postage Stamp Server tasks + +check.globals + +$LOGSUBDIR = $LOGDIR/pstamp +mkdir $LOGSUBDIR # these variables wil cycle through the known database names @@ -8,11 +12,13 @@ $pstampJob_DB = 0 $pstampFin_DB = 0 +$pstampRev_DB = 0 # set PS_DBSERVER if postage stamp database host is not the same as the value for DBSERVER in site.config -# warning: no quotes around the two words. it cause it to get passed to pstamptool as one word -# tricky to debug problem ensues +# warning: no quotes around the two words. That causes the variable to get passed to pstamptool as one word +# and a tricky to debug problem ensues # example: -# $PSDBSERVER = -dbserver hostname -if ($?PSDBSERVER == 0) +# $PS_DBSERVER = -dbserver hostname + +if ($?PS_DBSERVER == 0) $PS_DBSERVER = "" end @@ -51,5 +57,9 @@ active true end -end + task pstamp.job.revert + active true + end +end + macro pstamp.off @@ -74,4 +84,18 @@ end task pstamp.job.run + active false + end + task pstamp.job.revert + active false + end +end + +macro pstamp.revert.on + task pstamp.job.revert + active true + end +end +macro pstamp.revert.off + task pstamp.job.revert active false end @@ -87,4 +111,6 @@ task.exec + stdout $LOGSUBDIR/pstamp.request.find.log + stderr $LOGSUBDIR/pstamp.request.find.log if ($DB:n == 0) option DEFAULT @@ -124,4 +150,6 @@ task.exec + stdout $LOGSUBDIR/pstamp.request.load.log + stderr $LOGSUBDIR/pstamp.request.load.log $run = pstamptool -pendingreq if ($DB:n == 0) @@ -134,4 +162,5 @@ end add_poll_args run + # add_poll_labels run command $run end @@ -166,4 +195,6 @@ task.exec + stdout $LOGSUBDIR/pstamp.request.run.log + stderr $LOGSUBDIR/pstamp.request.run.log book npages pstampRequest -var N if ($N == 0) break @@ -221,13 +252,18 @@ task.exec + stdout $LOGSUBDIR/pstamp.finish.load.log + stderr $LOGSUBDIR/pstamp.finish.load.log + $run = pstamptool -completedreq if ($DB:n == 0) option DEFAULT - command pstamptool -completedreq else option $DB:$pstampFin_DB - command pstamptool -completedreq -dbname $DB:$pstampFin_DB $PS_DBSERVER + $run = $run -dbname $DB:$pstampFin_DB $PS_DBSERVER $pstampFin_DB ++ if ($pstampFin_DB >= $DB:n) set pstampFin_DB = 0 end + add_poll_args run + # add_poll_labels run + command $run end @@ -260,4 +296,6 @@ task.exec + stdout $LOGSUBDIR/request.finish.run.log + stderr $LOGSUBDIR/request.finish.run.log book npages pstampFinish -var N if ($N == 0) break @@ -315,13 +353,18 @@ task.exec + stdout $LOGSUBDIR/pstamp.job.load.log + stderr $LOGSUBDIR/pstamp.job.load.log + $run = pstamptool -pendingjob if ($DB:n == 0) option DEFAULT - command pstamptool -pendingjob -limit 10 else option $DB:$pstampJob_DB - command pstamptool -pendingjob -limit 10 -dbname $DB:$pstampJob_DB $PS_DBSERVER + $run = $run -dbname $DB:$pstampJob_DB $PS_DBSERVER $pstampJob_DB ++ if ($pstampJob_DB >= $DB:n) set pstampJob_DB = 0 end + add_poll_args run + # add_poll_labels run + command $run end @@ -338,15 +381,16 @@ end + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end + task.exit default showcommand failure end - task.exit crash - showcommand crash - end - - task.exit timeout - showcommand timeout - end end @@ -357,6 +401,12 @@ task.exec + stdout $LOGSUBDIR/pstamp.job.run.log + stderr $LOGSUBDIR/pstamp.job.run.log book npages pstampJob -var N - if ($N == 0) break + if ($N == 0) + periods -exec $RUNEXEC + break + end + periods -exec 0.05 book getpage pstampJob 0 -var pageName -key pantaskState INIT @@ -418,2 +468,44 @@ end end + +task pstamp.job.revert + host local + + periods -poll $LOADPOLL + periods -exec 10 + periods -timeout 300 + npending 1 + + task.exec + stdout $LOGSUBDIR/pstamp.job.revert.log + stderr $LOGSUBDIR/pstamp.job.revert.log + $run = pstamptool -revertjob -all + if ($DB:n == 0) + option DEFAULT + else + $run = $run $PS_DBSERVER -dbname $DB:$pstampRev_DB + $pstampRev_DB ++ + if ($pstampRev_DB >= $DB:n) set pstampRev_DB = 0 + end + add_poll_args run + # add_poll_labels run + command $run + end + + task.exit $EXIT_SUCCESS + # nothing to do + end + + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end +end + Index: branches/eam_branches/20090820/ippTasks/rcserver.pro =================================================================== --- branches/eam_branches/20090820/ippTasks/rcserver.pro (revision 25206) +++ branches/eam_branches/20090820/ippTasks/rcserver.pro (revision 25766) @@ -4,5 +4,5 @@ check.globals -#$LOGSUBDIR = $LOGDIR/rcserver +$LOGSUBDIR = $LOGDIR/rcserver mkdir $LOGSUBDIR @@ -66,5 +66,5 @@ stdout NULL - stderr $LOGDIR/rcserver.makefileset.load.log + stderr $LOGSUBDIR/rcserver.makefileset.load.log task.exec @@ -86,5 +86,5 @@ task.exit 0 # convert 'stdout' to book format - ipptool2book stdout rcPendingFS -key dist_id:prod_id -uniq -setword dbname $options:0 -setword pantaskState INIT + ipptool2book stdout rcPendingFS -key dist_id:dest_id -uniq -setword dbname $options:0 -setword pantaskState INIT if ($VERBOSE > 2) book listbook rcPendingFS @@ -129,7 +129,9 @@ book getword rcPendingFS $pageName stage -var STAGE book getword rcPendingFS $pageName stage_id -var STAGE_ID + book getword rcPendingFS $pageName label -var LABEL + book getword rcPendingFS $pageName filter -var FILTER book getword rcPendingFS $pageName dist_dir -var DIST_DIR # book getword rcPendingFS $pageName clean -var CLEAN - book getword rcPendingFS $pageName prod_id -var PROD_ID + book getword rcPendingFS $pageName dest_id -var DEST_ID book getword rcPendingFS $pageName product_name -var PRODUCT_NAME book getword rcPendingFS $pageName ds_dbhost -var DS_DBHOST @@ -141,5 +143,5 @@ host anyhost - sprintf logfile "%s/makefs.%s.%s.log" $DIST_DIR $DIST_ID $PROD_ID + sprintf logfile "%s/makefs.%s.%s.log" $DIST_DIR $DIST_ID $DEST_ID stdout $logfile stderr $logfile @@ -147,5 +149,5 @@ book setword rcPendingFS $pageName pantaskState RUN - $run = dist_make_fileset.pl --dist_id $DIST_ID --target_id $TARGET_ID --stage $STAGE --stage_id $STAGE_ID --prod_id $PROD_ID --product_name $PRODUCT_NAME --ds_dbhost $DS_DBHOST --ds_dbname $DS_DBNAME --dist_dir $DIST_DIR + $run = dist_make_fileset.pl --dist_id $DIST_ID --target_id $TARGET_ID --stage $STAGE --stage_id $STAGE_ID --label $LABEL --filter $FILTER --dest_id $DEST_ID --product_name $PRODUCT_NAME --ds_dbhost $DS_DBHOST --ds_dbname $DS_DBNAME --dist_dir $DIST_DIR add_standard_args run @@ -184,5 +186,5 @@ stdout NULL - stderr $LOGDIR/rcserver.checkstatus.load.log + stderr $LOGSUBDIR/rcserver.checkstatus.load.log task.exec @@ -241,9 +243,9 @@ stdout NULL - stderr $LOGDIR/rcserver.checkstatus.run.log + stderr $LOGSUBDIR/rcserver.checkstatus.run.log book setword rcPendingDest $pageName pantaskState RUN book getword rcPendingDest $pageName dest_id -var DEST_ID - book getword rcPendingDest $pageName prod_id -var PROD_ID + book getword rcPendingDest $pageName dest_id -var DEST_ID book getword rcPendingDest $pageName status_uri -var STATUS_URI book getword rcPendingDest $pageName last_fileset -var LAST_FILESET @@ -252,5 +254,5 @@ host anyhost - $run = rcserver_checkstatus.pl --dest_id $DEST_ID --prod_id $PROD_ID --status_uri $STATUS_URI --last_fileset $LAST_FILESET + $run = rcserver_checkstatus.pl --dest_id $DEST_ID --dest_id $DEST_ID --status_uri $STATUS_URI --last_fileset $LAST_FILESET add_standard_args run Index: branches/eam_branches/20090820/ippTasks/register.pro =================================================================== --- branches/eam_branches/20090820/ippTasks/register.pro (revision 25206) +++ branches/eam_branches/20090820/ippTasks/register.pro (revision 25766) @@ -217,4 +217,6 @@ book getword regPendingImfile $pageName tmp_class_id -var TMP_CLASS_ID book getword regPendingImfile $pageName uri -var URI + book getword regPendingImfile $pageName bytes -var BYTES + book getword regPendingImfile $pageName md5sum -var MD5SUM book getword regPendingImfile $pageName workdir -var WORKDIR_TEMPLATE book getword regPendingImfile $pageName dbname -var DBNAME @@ -244,5 +246,5 @@ # XXX register_imfile.pl differs from the standard script : it does not have an 'outroot' argument, and it does not take '--redirect' - $run = register_imfile.pl --exp_id $EXP_ID --tmp_class_id $TMP_CLASS_ID --tmp_exp_name $TMP_EXP_NAME --uri $URI --logfile $logfile + $run = register_imfile.pl --exp_id $EXP_ID --tmp_class_id $TMP_CLASS_ID --tmp_exp_name $TMP_EXP_NAME --uri $URI --logfile $logfile --bytes $BYTES --md5sum $MD5SUM add_standard_args run Index: branches/eam_branches/20090820/ippTasks/simtest.pro =================================================================== --- branches/eam_branches/20090820/ippTasks/simtest.pro (revision 25206) +++ branches/eam_branches/20090820/ippTasks/simtest.pro (revision 25766) @@ -85,7 +85,10 @@ ## XXX this will need to be optional as well + ## we could use the book to find OBSTYPE = OBJECT, get CENTER.RA, CENTER.DEC + ## or else we could have ppSimSequence generate an appropriate skycells call file TESS/Images.dat found if ($found == 0) - exec skycells -mode square -level 8 -scale 0.2 -D CATDIR TESS + # this command is approriate only to the dither pattern and camera define in simtest.basic.config + exec skycells -mode local -scale 0.2 -center 270.75 -23.70 -size 0.3 0.3 -nx 3 -ny 3 -D CATDIR TESS end end Index: branches/eam_branches/20090820/ippTasks/site.mhpcc.pro =================================================================== --- branches/eam_branches/20090820/ippTasks/site.mhpcc.pro (revision 25206) +++ branches/eam_branches/20090820/ippTasks/site.mhpcc.pro (revision 25766) @@ -14,4 +14,5 @@ controller host add ipp023 controller host add ipp024 + controller host add ipp025 controller host add ipp026 controller host add ipp028 @@ -39,5 +40,5 @@ controller host add ipp051 controller host add ipp052 - controller host add ipp053 +# controller host add ipp053 end @@ -65,5 +66,5 @@ if ("$1" == "on") $NEBULOUS = 1 - $default_host = ipp023 + $default_host = any $workdir_template = neb://@HOST@.0 else Index: branches/eam_branches/20090820/ippTasks/summit.copy.pro =================================================================== --- branches/eam_branches/20090820/ippTasks/summit.copy.pro (revision 25206) +++ branches/eam_branches/20090820/ippTasks/summit.copy.pro (revision 25766) @@ -380,6 +380,10 @@ periods -poll 0.05 periods -timeout 650 - # trage 16:00 23:59 - # trage 00:00 04:00 + trange -reset + # only active in the night (18:00 to 06:00 HST, times are UT): + # trange 04:00 16:00 + # only active in the day (06:00 to 18:00 HST, times are UT): + # trange 16:00 23:59 + # trange 00:00 04:00 task.exec @@ -449,4 +453,5 @@ end if (("$MD5SUM" != "NULL") && ("$MD5SUM" != "0") && (not($COMPRESS))) +# && (($YEAR > 2008) || (("$YEAR" = "2007") && ($MONTH > 8)))) $run = $run --md5 $MD5SUM end Index: branches/eam_branches/20090820/ippTasks/warp.pro =================================================================== --- branches/eam_branches/20090820/ippTasks/warp.pro (revision 25206) +++ branches/eam_branches/20090820/ippTasks/warp.pro (revision 25766) @@ -49,4 +49,10 @@ active true end + task warp.revert.overlap + active true + end + task warp.revert.warped + active true + end end @@ -66,4 +72,10 @@ end task warp.advancerun + active false + end + task warp.revert.overlap + active false + end + task warp.revert.warped active false end Index: branches/eam_branches/20090820/ippTools/configure.ac =================================================================== --- branches/eam_branches/20090820/ippTools/configure.ac (revision 25206) +++ branches/eam_branches/20090820/ippTools/configure.ac (revision 25766) @@ -18,5 +18,5 @@ PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0]) PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) -PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.54]) +PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.55]) PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}" Index: branches/eam_branches/20090820/ippTools/doc/addstar_flow.txt =================================================================== --- branches/eam_branches/20090820/ippTools/doc/addstar_flow.txt (revision 25766) +++ branches/eam_branches/20090820/ippTools/doc/addstar_flow.txt (revision 25766) @@ -0,0 +1,34 @@ +### Set up the addRun row during the previous stage +#>> camtool.c @ addprocessedexpMode(pxConfig *config) +camtool -cam_id $cam_id -addprocessedexp ... + +### Catch that things now need to be added. This should check for magic_status as well, probably. +#>> add.pro / addtool.c / addtoolConfig.c +addtool -pendingsmf +#addMe METADATA +# add_id S64 5 +#END + +### Run the addstar script. +#>> addstar_run.pl +addstar_run.pl --add_id $add_id --camera $camera --camroot=$camroot --dvodb $DVODB + addtool -pendingsmf -add_id $add_id +#addMe METADATA +# add_id S64 5 +# workdir STR /path/to/workdir +# +#END + + ppConfigDump -camera $camera -recipe ADDSTAR $recipt_addstar -dump-recipe ADDSTAR - + addstar -D CAMERA $camdir -update -D CATDIR $dvodbReal $realFile + addtool -add_id $add_id -addprocessedsmf ... + addtool -add_id $add_id -updaterun -state full + +### Reverting +#>> addtool.c +addtool -revertprocessedsmf ... + + + + + Index: branches/eam_branches/20090820/ippTools/share/Makefile.am =================================================================== --- branches/eam_branches/20090820/ippTools/share/Makefile.am (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/Makefile.am (revision 25766) @@ -4,4 +4,8 @@ dist_pkgdata_DATA = \ + addtool_find_pendingexp.sql \ + addtool_queue_cam_id.sql \ + addtool_reset_faulted_runs.sql \ + addtool_revertprocessedexp.sql \ camtool_donecleanup.sql \ camtool_find_chip_id.sql \ @@ -43,4 +47,5 @@ dettool_normalizedstat.sql \ dettool_pending.sql \ + dettool_pendingcleanup_detrunsummary.sql \ dettool_pendingcleanup_normalizedexp.sql \ dettool_pendingcleanup_normalizedimfile.sql \ @@ -105,5 +110,13 @@ disttool_definebyquery_stack.sql \ disttool_definebyquery_warp.sql \ - disttool_pendingcomponent.sql \ + disttool_defineinterest.sql \ + disttool_listinterests.sql \ + disttool_pending_camera.sql \ + disttool_pending_chip.sql \ + disttool_pending_diff.sql \ + disttool_pending_fake.sql \ + disttool_pending_raw.sql \ + disttool_pending_stack.sql \ + disttool_pending_warp.sql \ disttool_pendingfileset.sql \ disttool_pendingdest.sql \ @@ -111,6 +124,6 @@ disttool_queuercrun.sql \ disttool_revertrcrun.sql \ - disttool_revertrun_update.sql \ - disttool_revertrun_delete.sql \ + disttool_revertrun.sql \ + disttool_revertcomponent.sql \ disttool_revertfileset.sql \ disttool_toadvance.sql \ @@ -137,9 +150,9 @@ flatcorr_dropcamera.sql \ magictool_addmask.sql \ - magictool_censor_raw.sql \ - magictool_censor_chip.sql \ - magictool_censor_camera.sql \ - magictool_censor_warp.sql \ - magictool_censor_diff.sql \ + magictool_restore_camera.sql \ + magictool_restore_chip.sql \ + magictool_restore_diff.sql \ + magictool_restore_raw.sql \ + magictool_restore_warp.sql \ magictool_create_tmp_warpcomplete.sql \ magictool_definebyquery_insert.sql \ @@ -160,4 +173,5 @@ magictool_revertnode.sql \ magicdstool_completed_runs.sql \ + magicdstool_completedrevert.sql \ magicdstool_definebyquery_raw.sql \ magicdstool_definebyquery_chip.sql \ @@ -167,7 +181,11 @@ magicdstool_getrunids.sql \ magicdstool_getskycells.sql \ - magicdstool_todestreak.sql \ + magicdstool_revertdestreakedfile.sql \ + magicdstool_todestreak_camera.sql \ + magicdstool_todestreak_chip.sql \ + magicdstool_todestreak_diff.sql \ + magicdstool_todestreak_raw.sql \ + magicdstool_todestreak_warp.sql \ magicdstool_toremove.sql \ - magicdstool_torestore.sql \ magicdstool_torevert_raw.sql \ magicdstool_torevert_chip.sql \ @@ -175,8 +193,13 @@ magicdstool_torevert_warp.sql \ magicdstool_torevert_diff.sql \ + pstamptool_completedreq.sql \ pstamptool_datastore.sql \ + pstamptool_listjob.sql \ pstamptool_pendingjob.sql \ pstamptool_pendingreq.sql \ pstamptool_project.sql \ + pstamptool_revertjob.sql \ + pstamptool_revertreq.sql \ + pstamptool_revertreq_deletejobs.sql \ pxadmin_create_tables.sql \ pxadmin_create_mirror_tables.sql \ Index: branches/eam_branches/20090820/ippTools/share/addtool_find_pendingexp.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/addtool_find_pendingexp.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/addtool_find_pendingexp.sql (revision 25766) @@ -0,0 +1,26 @@ +SELECT + addRun.*, + rawExp.exp_tag, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel +FROM addRun +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + USING(chip_id) +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id +LEFT JOIN addProcessedExp + USING(add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + camRun.state = 'full' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addMask.label IS NULL + Index: branches/eam_branches/20090820/ippTools/share/addtool_queue_cam_id.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/addtool_queue_cam_id.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/addtool_queue_cam_id.sql (revision 25766) @@ -0,0 +1,14 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + cam_id, -- cam_id + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- label + '%s', -- dvodb + 0 -- magicked + FROM camRun + WHERE + camRun.state = 'full' + AND camRun.cam_id = %lld Index: branches/eam_branches/20090820/ippTools/share/addtool_reset_faulted_runs.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/addtool_reset_faulted_runs.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/addtool_reset_faulted_runs.sql (revision 25766) @@ -0,0 +1,8 @@ +UPDATE addRun, addProcessedExp, camRun, chipRun, rawExp +SET addRun.state = 'new' +WHERE + addRun.add_id = addProcessedExp.add_id + AND addRun.cam_id = camRun.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND addProcessedExp.fault != 0 Index: branches/eam_branches/20090820/ippTools/share/addtool_revertprocessedexp.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/addtool_revertprocessedexp.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/addtool_revertprocessedexp.sql (revision 25766) @@ -0,0 +1,8 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun, camRun, chipRun, rawExp +WHERE + addRun.add_id = addProcessedExp.add_id + AND addRun.cam_id = camRun.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND addProcessedExp.fault != 0 Index: branches/eam_branches/20090820/ippTools/share/camtool_pendingcleanupexp.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/camtool_pendingcleanupexp.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/camtool_pendingcleanupexp.sql (revision 25766) @@ -13,4 +13,4 @@ USING(cam_id) WHERE - (camRun.state = 'goto_cleaned' OR camRun.state = 'goto_purged') + (camRun.state = 'goto_cleaned' OR camRun.state = 'goto_purged' OR camRun.state = 'goto_scrubbed') Index: branches/eam_branches/20090820/ippTools/share/camtool_pendingcleanuprun.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/camtool_pendingcleanuprun.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/camtool_pendingcleanuprun.sql (revision 25766) @@ -9,3 +9,3 @@ USING (exp_id) WHERE - (camRun.state = 'goto_cleaned' OR camRun.state = 'goto_purged') + (camRun.state = 'goto_cleaned' OR camRun.state = 'goto_purged' OR camRun.state = 'goto_scrubbed') Index: branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_normalizedexp.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_normalizedexp.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_normalizedexp.sql (revision 25766) @@ -0,0 +1,11 @@ +SELECT + detNormalizedExp.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detNormalizedExp + USING(det_id,iteration) +WHERE + (detNormalizedExp.data_state = 'cleaned' + OR detNormalizedExp.data_state = 'scrubbed' + OR detNormalizedExp.data_state = 'purged') + Index: branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_normalizedimfile.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_normalizedimfile.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_normalizedimfile.sql (revision 25766) @@ -0,0 +1,11 @@ +SELECT + detNormalizedImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detNormalizedImfile + USING(det_id,iteration) +WHERE + (detNormalizedImfile.data_state = 'cleaned' + OR detNormalizedImfile.data_state = 'scrubbed' + OR detNormalizedImfile.data_state = 'purged') + Index: branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_normalizedstat.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_normalizedstat.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_normalizedstat.sql (revision 25766) @@ -0,0 +1,11 @@ +SELECT + detNormalizedStatImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detNormalizedStatImfile + USING(det_id,iteration) +WHERE + (detNormalizedStatImfile.data_state = 'cleaned' + OR detNormalizedStatImfile.data_state = 'scrubbed' + OR detNormalizedStatImfile.data_state = 'purged') + Index: branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_processedexp.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_processedexp.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_processedexp.sql (revision 25766) @@ -0,0 +1,11 @@ +-- need to restrict to a single detRunSummary (require all to say 'cleaned'?) +SELECT + detProcessedExp.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detProcessedExp + USING(det_id) +WHERE + (detProcessedExp.data_state = 'cleaned' + OR detProcessedExp.data_state = 'scrubbed' + OR detProcessedExp.data_state = 'purged') Index: branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_processedimfile.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_processedimfile.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_processedimfile.sql (revision 25766) @@ -0,0 +1,11 @@ +-- need to restrict to a single detRunSummary (require all to say 'cleaned'?) +SELECT + detProcessedImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detProcessedImfile + USING(det_id) +WHERE + (detProcessedImfile.data_state = 'cleaned' + OR detProcessedImfile.data_state = 'scrubbed' + OR detProcessedImfile.data_state = 'purged') Index: branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_residexp.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_residexp.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_residexp.sql (revision 25766) @@ -0,0 +1,10 @@ +SELECT + detResidExp.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detResidExp + USING(det_id,iteration) +WHERE + (detResidExp.data_state = 'cleaned' + OR detResidExp.data_state = 'scrubbed' + OR detResidExp.data_state = 'purged') Index: branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_residimfile.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_residimfile.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_residimfile.sql (revision 25766) @@ -0,0 +1,12 @@ +SELECT + detResidImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detResidImfile + USING(det_id,iteration) +WHERE + (detResidImfile.data_state = 'cleaned' + OR detResidImfile.data_state = 'scrubbed' + OR detResidImfile.data_state = 'purged') + + Index: branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_stacked.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_stacked.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/dettool_donecleanup_stacked.sql (revision 25766) @@ -0,0 +1,10 @@ +SELECT + detStackedImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detStackedImfile + USING(det_id,iteration) +WHERE + (detStackedImfile.data_state = 'cleaned' + OR detStackedImfile.data_state = 'scrubbed' + OR detStackedImfile.data_state = 'purged') Index: branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_detrunsummary.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_detrunsummary.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_detrunsummary.sql (revision 25766) @@ -0,0 +1,42 @@ +SELECT DISTINCT detRunSummary.* +FROM detRunSummary + JOIN detProcessedImfile USING (det_id) + JOIN detProcessedExp USING (det_id) + JOIN detStackedImfile USING (det_id,iteration) + JOIN detResidExp USING (det_id,iteration) + JOIN detResidImfile USING (det_id,iteration) + JOIN detNormalizedStatImfile USING (det_id,iteration) + JOIN detNormalizedImfile USING (det_id,iteration) + JOIN detNormalizedExp USING (det_id,iteration) +WHERE (!( + ( + detRunSummary.data_state = 'full' OR + detRunSummary.data_state = 'cleaned' + ) OR + ( + !((detProcessedImfile.data_state = 'cleaned' OR + detProcessedImfile.data_state = 'scrubbed' OR + detProcessedImfile.data_state = 'purged') AND + (detProcessedExp.data_state = 'cleaned' OR + detProcessedExp.data_state = 'scrubbed' OR + detProcessedExp.data_state = 'purged') AND + (detStackedImfile.data_state = 'cleaned' OR + detStackedImfile.data_state = 'scrubbed' OR + detStackedImfile.data_state = 'purged') AND + (detResidExp.data_state = 'cleaned' OR + detResidExp.data_state = 'scrubbed' OR + detResidExp.data_state = 'purged') AND + (detResidImfile.data_state = 'cleaned' OR + detResidImfile.data_state = 'scrubbed' OR + detResidImfile.data_state = 'purged') AND + (detNormalizedStatImfile.data_state = 'cleaned' OR + detNormalizedStatImfile.data_state = 'scrubbed' OR + detNormalizedStatImfile.data_state = 'purged') AND + (detNormalizedImfile.data_state = 'cleaned' OR + detNormalizedImfile.data_state = 'scrubbed' OR + detNormalizedImfile.data_state = 'purged') AND + (detNormalizedExp.data_state = 'cleaned' OR + detNormalizedExp.data_state = 'scrubbed' OR + detNormalizedExp.data_state = 'purged') + ) + ))) Index: branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_normalizedexp.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_normalizedexp.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_normalizedexp.sql (revision 25766) @@ -1,9 +1,21 @@ SELECT detNormalizedExp.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detNormalizedExp USING(det_id,iteration) +JOIN detInputExp + USING(det_id,iteration) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detNormalizedExp.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detNormalizedExp.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detNormalizedExp.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detNormalizedExp.data_state = 'full') + OR detNormalizedExp.data_state = 'goto_cleaned' + OR detNormalizedExp.data_state = 'goto_scrubbed' + OR detNormalizedExp.data_state = 'goto_purged') + Index: branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_normalizedimfile.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_normalizedimfile.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_normalizedimfile.sql (revision 25766) @@ -1,9 +1,21 @@ -SELECT +SELECT DISTINCT detNormalizedImfile.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detNormalizedImfile USING(det_id,iteration) +JOIN detInputExp + USING(det_id,iteration) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detNormalizedImfile.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detNormalizedImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detNormalizedImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detNormalizedImfile.data_state = 'full') + OR detNormalizedImfile.data_state = 'goto_cleaned' + OR detNormalizedImfile.data_state = 'goto_scrubbed' + OR detNormalizedImfile.data_state = 'goto_purged') + Index: branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_normalizedstat.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_normalizedstat.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_normalizedstat.sql (revision 25766) @@ -1,9 +1,21 @@ -SELECT +SELECT DISTINCT detNormalizedStatImfile.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detNormalizedStatImfile USING(det_id,iteration) +JOIN detInputExp + USING(det_id,iteration) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detNormalizedStatImfile.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detNormalizedStatImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detNormalizedStatImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detNormalizedStatImfile.data_state = 'full') + OR detNormalizedStatImfile.data_state = 'goto_cleaned' + OR detNormalizedStatImfile.data_state = 'goto_scrubbed' + OR detNormalizedStatImfile.data_state = 'goto_purged') + Index: branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_processedexp.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_processedexp.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_processedexp.sql (revision 25766) @@ -2,9 +2,18 @@ SELECT detProcessedExp.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detProcessedExp USING(det_id) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detProcessedExp.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detProcessedExp.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detProcessedExp.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detProcessedExp.data_state = 'full') + OR detProcessedExp.data_state = 'goto_cleaned' + OR detProcessedExp.data_state = 'goto_scrubbed' + OR detProcessedExp.data_state = 'goto_purged') Index: branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_processedimfile.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_processedimfile.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_processedimfile.sql (revision 25766) @@ -1,10 +1,15 @@ --- need to restrict to a single detRunSummary (require all to say 'cleaned'?) SELECT detProcessedImfile.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detProcessedImfile USING(det_id) +JOIN detInputExp + USING(det_id,iteration,exp_id) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detProcessedImfile.data_state = 'full' + (detProcessedImfile.data_state = 'goto_cleaned' + OR detProcessedImfile.data_state = 'goto_scrubbed' + OR detProcessedImfile.data_state = 'goto_purged') + Index: branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_residexp.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_residexp.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_residexp.sql (revision 25766) @@ -1,9 +1,20 @@ SELECT detResidExp.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detResidExp USING(det_id,iteration) +JOIN detInputExp + USING(det_id,iteration,exp_id) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detResidExp.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detResidExp.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detResidExp.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detResidExp.data_state = 'full') + OR detResidExp.data_state = 'goto_cleaned' + OR detResidExp.data_state = 'goto_scrubbed' + OR detResidExp.data_state = 'goto_purged') Index: branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_residimfile.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_residimfile.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_residimfile.sql (revision 25766) @@ -1,9 +1,22 @@ SELECT detResidImfile.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detResidImfile USING(det_id,iteration) +JOIN detInputExp + USING(det_id,iteration,exp_id) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detResidImfile.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detResidImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detResidImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detResidImfile.data_state = 'full') + OR detResidImfile.data_state = 'goto_cleaned' + OR detResidImfile.data_state = 'goto_scrubbed' + OR detResidImfile.data_state = 'goto_purged') + + Index: branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_stacked.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_stacked.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/dettool_pendingcleanup_stacked.sql (revision 25766) @@ -1,9 +1,20 @@ -SELECT +SELECT DISTINCT detStackedImfile.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detStackedImfile - USING(det_id,iteration) + USING(det_id,iteration) +JOIN detInputExp + USING(det_id,iteration) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detStackedImfile.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detStackedImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detStackedImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detStackedImfile.data_state = 'full') + OR detStackedImfile.data_state = 'goto_cleaned' + OR detStackedImfile.data_state = 'goto_scrubbed' + OR detStackedImfile.data_state = 'goto_purged') Index: branches/eam_branches/20090820/ippTools/share/difftool_completed_runs.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/difftool_completed_runs.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/difftool_completed_runs.sql (revision 25766) @@ -1,19 +1,16 @@ -SELECT DISTINCT - diff_id, - all_magicked as magicked -FROM ( - SELECT - COUNT(diffInputSkyfile.skycell_id), COUNT(diffSkyfile.skycell_id), - diffSkyfile.*, - SUM(!diffSkyfile.magicked) = 0 as all_magicked - FROM diffRun - JOIN diffInputSkyfile USING(diff_id) - LEFT JOIN diffSkyfile USING(diff_id, skycell_id) - WHERE - diffRun.state = 'new' - GROUP BY - diffInputSkyfile.diff_id - HAVING - COUNT(diffInputSkyfile.skycell_id) = COUNT(diffSkyfile.skycell_id) - AND SUM(diffSkyfile.fault) = 0 - ) as Foo +SELECT + COUNT(diffInputSkyfile.skycell_id), + COUNT(diffSkyfile.skycell_id), + diffSkyfile.*, + SUM(!diffSkyfile.magicked) = 0 as all_magicked +FROM diffRun +JOIN diffInputSkyfile USING(diff_id) +LEFT JOIN diffSkyfile USING(diff_id, skycell_id) +WHERE + diffRun.state = 'new' +-- WHERE hook %s +GROUP BY + diffInputSkyfile.diff_id +HAVING + COUNT(diffInputSkyfile.skycell_id) = COUNT(diffSkyfile.skycell_id) + AND SUM(diffSkyfile.fault) = 0 Index: branches/eam_branches/20090820/ippTools/share/difftool_definewarpstack_part1.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/difftool_definewarpstack_part1.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/difftool_definewarpstack_part1.sql (revision 25766) @@ -27,5 +27,6 @@ ) AS diffExp USING(exp_id, warp_id) WHERE - warpSkyfile.fault = 0 + warpRun.state = 'full' + AND warpSkyfile.fault = 0 AND warpSkyfile.quality = 0 -- warp where hook %s Index: branches/eam_branches/20090820/ippTools/share/difftool_definewarpstack_part2.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/difftool_definewarpstack_part2.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/difftool_definewarpstack_part2.sql (revision 25766) @@ -30,4 +30,5 @@ WHERE warpSkyfile.warp_id = %lld + AND warpRun.state = 'full' AND warpSkyfile.fault = 0 AND warpSkyfile.quality = 0 Index: branches/eam_branches/20090820/ippTools/share/difftool_skyfile.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/difftool_skyfile.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/difftool_skyfile.sql (revision 25766) @@ -12,4 +12,5 @@ -- The following are only valid for warps -- XXX This needs to be more clever to handle diffs between stacks + -- Zero points are appropriate for both forward and backward diffs camProcessedInput.zpt_obs, camProcessedInput.zpt_stdev, @@ -19,7 +20,17 @@ rawInput.camera, rawInput.exp_name AS exp_name_1, + rawInput.exp_id AS exp_id_1, + chipInput.chip_id AS chip_id_1, + camInput.cam_id AS cam_id_1, + fakeInput.fake_id AS fake_id_1, + camProcessedInput.sigma_ra AS sigma_ra_1, + camProcessedInput.sigma_dec AS sigma_dec_1, rawTemplate.exp_name AS exp_name_2, - rawInput.exp_id AS exp_id_1, - rawTemplate.exp_id AS exp_id_2 + rawTemplate.exp_id AS exp_id_2, + chipTemplate.chip_id AS chip_id_2, + camTemplate.cam_id AS cam_id_2, + fakeTemplate.fake_id AS fake_id_2, + camProcessedTemplate.sigma_ra AS sigma_ra_2, + camProcessedTemplate.sigma_dec AS sigma_dec_2 FROM diffRun JOIN diffSkyfile USING(diff_id) @@ -43,4 +54,6 @@ LEFT JOIN camRun AS camTemplate ON camTemplate.cam_id = fakeTemplate.cam_id +LEFT JOIN camProcessedExp AS camProcessedTemplate + ON camProcessedTemplate.cam_id = camTemplate.cam_id LEFT JOIN chipRun AS chipTemplate ON chipTemplate.chip_id = camTemplate.chip_id Index: branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_camera.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_camera.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_camera.sql (revision 25766) @@ -3,4 +3,5 @@ camRun.cam_id as stage_id, rawExp.exp_name as run_tag, + camRun.magicked, distTarget.label, distTarget.target_id, Index: branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_chip.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_chip.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_chip.sql (revision 25766) @@ -2,4 +2,5 @@ 'chip' as stage, chipRun.chip_id as stage_id, + chipRun.magicked, rawExp.exp_name as run_tag, chipRun.label, Index: branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_diff.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_diff.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_diff.sql (revision 25766) @@ -2,4 +2,5 @@ 'diff' as stage, diffRun.diff_id AS stage_id, + diffRun.magicked, rawExp.exp_name as run_tag, distTarget.label, Index: branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_fake.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_fake.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_fake.sql (revision 25766) @@ -2,4 +2,5 @@ 'fake' as stage, fakeRun.fake_id AS stage_id, + CAST(0 AS SIGNED) AS magicked, rawExp.exp_name as run_tag, distTarget.label, Index: branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_raw.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_raw.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_raw.sql (revision 25766) @@ -3,4 +3,5 @@ rawExp.exp_id AS stage_id, rawExp.exp_name AS run_tag, + rawExp.magicked, distTarget.label, distTarget.target_id, Index: branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_stack.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_stack.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_stack.sql (revision 25766) @@ -2,4 +2,5 @@ 'stack' as stage, stackRun.stack_id AS stage_id, + CAST(0 AS SIGNED) AS magicked, -- run tag in the form 'stack.$skycell_id.$stack_id' CONCAT_WS('.', 'stack', stackRun.skycell_id, convert(stackRun.stack_id, CHAR)) as run_tag, @@ -8,4 +9,5 @@ distTarget.clean FROM stackRun +JOIN stackSumSkyfile USING(stack_id) JOIN distTarget ON distTarget.stage = 'stack' AND stackRun.label = distTarget.label @@ -19,2 +21,4 @@ AND distRun.dist_id IS NULL AND ((stackRun.state = 'full') OR (distTarget.clean AND stackRun.state = 'cleaned')) + -- we shouldn't need to check fault. If faulted it shouldn't be full + AND (stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0) Index: branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_warp.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_warp.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/disttool_definebyquery_warp.sql (revision 25766) @@ -2,4 +2,5 @@ 'warp' as stage, warpRun.warp_id AS stage_id, + warpRun.magicked, rawExp.exp_name as run_tag, distTarget.label, Index: branches/eam_branches/20090820/ippTools/share/disttool_defineinterest.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_defineinterest.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/disttool_defineinterest.sql (revision 25766) @@ -0,0 +1,6 @@ +INSERT INTO rcInterest + SELECT 0, dest_id, target_id, '@STATE@' + FROM distTarget + JOIN rcDestination + LEFT JOIN rcInterest USING(dest_id, target_id) + WHERE (int_id IS NULL) Index: branches/eam_branches/20090820/ippTools/share/disttool_listinterests.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_listinterests.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/disttool_listinterests.sql (revision 25766) @@ -0,0 +1,13 @@ +SELECT + int_id, + rcInterest.state AS state, + dest_id, + name AS dest_name, + rcDestination.state as dest_state, + label, + filter, + stage, + distTarget.state as target_state +FROM rcInterest + JOIN distTarget USING(target_id) + JOIN rcDestination USING(dest_id) Index: branches/eam_branches/20090820/ippTools/share/disttool_pending_camera.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_pending_camera.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/disttool_pending_camera.sql (revision 25766) @@ -0,0 +1,31 @@ +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + 'exposure' AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + camProcessedExp.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + camRun.state, + camRun.state AS data_state, + camProcessedExp.quality, + distRun.no_magic, + chipRun.magicked +FROM distRun +JOIN camRun ON camRun.cam_id = distRun.stage_id +JOIN camProcessedExp USING(cam_id) +JOIN chipRun USING(chip_id) +-- JOIN chipProcessedImfile USING(exp_id, chip_id) +JOIN rawExp using(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id +-- AND chipProcessedImfile.class_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'camera' + AND distComponent.dist_id IS NULL + AND (distRun.clean OR (chipRun.magicked AND camRun.magicked) OR distRun.no_magic) + AND (camRun.state = 'full' OR (distRun.clean AND camRun.state = 'cleaned')) Index: branches/eam_branches/20090820/ippTools/share/disttool_pending_chip.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_pending_chip.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/disttool_pending_chip.sql (revision 25766) @@ -0,0 +1,29 @@ +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + chipProcessedImfile.class_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + chipProcessedImfile.path_base, + chipProcessedImfile.path_base as chip_path_base, + chipRun.state, + chipProcessedImfile.data_state, + chipProcessedImfile.quality, + distRun.no_magic, + chipProcessedImfile.magicked +FROM distRun +JOIN chipRun ON chipRun.chip_id = distRun.stage_id +JOIN rawExp using(exp_id) +JOIN chipProcessedImfile ON chipProcessedImfile.chip_id = stage_id +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND chipProcessedImfile.class_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'chip' + AND distComponent.dist_id IS NULL + AND (distRun.clean OR chipRun.magicked OR distRun.no_magic) + AND (chipRun.state = 'full' OR (distRun.clean AND chipRun.state = 'cleaned')) Index: branches/eam_branches/20090820/ippTools/share/disttool_pending_diff.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_pending_diff.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/disttool_pending_diff.sql (revision 25766) @@ -0,0 +1,37 @@ +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + diffSkyfile.skycell_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + diffSkyfile.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + diffRun.state, + -- data_state doesn't exist yet + -- diffSkyfile.data_state, + 'full' AS data_state, + diffSkyfile.quality, + distRun.no_magic, + diffSkyfile.magicked +FROM distRun +JOIN diffRun ON stage_id = diff_id +JOIN diffSkyfile using(diff_id) +JOIN diffInputSkyfile USING(diff_id, skycell_id) +JOIN warpRun + ON warpRun.warp_id = diffInputSkyfile.warp1 +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND diffSkyfile.skycell_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'diff' + AND distComponent.dist_id IS NULL + AND (distRun.clean OR diffRun.magicked OR distRun.no_magic) + AND (diffRun.state = 'full' OR (distRun.clean AND diffRun.state = 'cleaned')) Index: branches/eam_branches/20090820/ippTools/share/disttool_pending_fake.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_pending_fake.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/disttool_pending_fake.sql (revision 25766) @@ -0,0 +1,29 @@ +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + fakeProcessedImfile.class_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + fakeProcessedImfile.path_base, + CAST(NULL AS CHAR(255)) AS chip_path_base, + fakeRun.state, + fakeRun.state AS data_state, + 0 as quality, + distRun.no_magic, + 0 +FROM distRun +JOIN fakeRun ON fakeRun.fake_id = distRun.stage_id +JOIN fakeProcessedImfile USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id, exp_id) +JOIN rawExp using(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND fakeProcessedImfile.class_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'fake' + AND distComponent.dist_id IS NULL Index: branches/eam_branches/20090820/ippTools/share/disttool_pending_raw.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_pending_raw.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/disttool_pending_raw.sql (revision 25766) @@ -0,0 +1,58 @@ +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + rawImfile.class_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + rawImfile.uri as path_base, -- change this once rawImfile has path_base + chipProcessedImfile.path_base as chip_path_base, + CAST(NULL AS CHAR(255)) AS state, + CAST(NULL AS CHAR(255)) AS data_state, + 0 as quality, + distRun.no_magic, + rawImfile.magicked +FROM distRun +JOIN rawExp ON rawExp.exp_id = stage_id +JOIN rawImfile USING(exp_id) +JOIN chipRun ON chipRun.exp_id = rawExp.exp_id and chipRun.magicked +JOIN chipProcessedImfile + USING(chip_id, class_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND rawImfile.class_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.clean = 0 + AND distRun.stage = 'raw' + AND distComponent.dist_id IS NULL + AND (rawExp.magicked OR distRun.no_magic) +UNION + -- raw stage clean (dbinfo only) +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + 'exposure' AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + CAST(NULL AS CHAR(255)) AS path_base, + CAST(NULL AS CHAR(255)) AS chip_path_base, + CAST(NULL AS CHAR(255)) AS state, + CAST(NULL AS CHAR(255)) AS data_state, + 0 as quality, + distRun.no_magic, + rawExp.magicked +FROM distRun +JOIN rawExp ON exp_id = stage_id +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id +WHERE + distRun.state = 'new' + AND distRun.stage = 'raw' + AND distRun.clean + AND distComponent.dist_id IS NULL Index: branches/eam_branches/20090820/ippTools/share/disttool_pending_stack.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_pending_stack.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/disttool_pending_stack.sql (revision 25766) @@ -0,0 +1,48 @@ +SELECT DISTINCT + distRun.dist_id, + distRun.label, + stage, + stage_id, + stackRun.skycell_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + stackSumSkyfile.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + stackRun.state, + -- data_state doesn't exist yet + -- stackSumSkyfile.data_state, + 'full' AS data_state, + stackSumSkyfile.quality, + 1 AS no_magic, + 0 AS magicked +FROM distRun +JOIN stackRun + ON stage_id = stack_id +JOIN stackSumSkyfile + USING(stack_id) +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpSkyfile + ON stackInputSkyfile.warp_id = warpSkyfile.warp_id + AND stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.tess_id = warpSkyfile.tess_id +JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN rawExp + USING (exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND stackRun.skycell_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'stack' + AND distComponent.dist_id IS NULL + AND (stackRun.state = 'full' OR (distRun.clean AND stackRun.state = 'cleaned')) + AND (stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0) Index: branches/eam_branches/20090820/ippTools/share/disttool_pending_warp.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_pending_warp.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/disttool_pending_warp.sql (revision 25766) @@ -0,0 +1,32 @@ +SELECT + distRun.dist_id, + distRun.label, + stage, + stage_id, + warpSkyfile.skycell_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + warpSkyfile.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + warpRun.state, + warpSkyfile.data_state, + warpSkyfile.quality, + distRun.no_magic, + warpSkyfile.magicked +FROM distRun +JOIN warpRun ON stage_id = warp_id +JOIN warpSkyfile using(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun ON camRun.chip_id = chipRun.chip_id +JOIN rawExp using(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND warpSkyfile.skycell_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'warp' + AND distComponent.dist_id IS NULL + AND (distRun.clean OR warpRun.magicked OR distRun.no_magic) + AND (warpRun.state = 'full' OR (distRun.clean AND warpRun.state = 'cleaned')) Index: branches/eam_branches/20090820/ippTools/share/disttool_pendingcomponent.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_pendingcomponent.sql (revision 25206) +++ (revision ) @@ -1,283 +1,0 @@ -SELECT * FROM ( - -- raw stage -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - rawImfile.class_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - rawImfile.uri as path_base, -- change this once rawImfile has path_base - chipProcessedImfile.path_base as chip_path_base, - NULL as state, - NULL as data_state, - 0 as quality, - distRun.no_magic, - rawImfile.magicked -FROM distRun -JOIN rawExp ON rawExp.exp_id = stage_id -JOIN rawImfile USING(exp_id) -JOIN chipRun ON chipRun.exp_id = rawExp.exp_id and chipRun.magicked -JOIN chipProcessedImfile - USING(chip_id, class_id) -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id - AND rawImfile.class_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.clean = 0 - AND distRun.stage = 'raw' - AND distComponent.dist_id IS NULL - AND (rawExp.magicked OR distRun.no_magic) - -- where hook 1 %s -UNION - -- raw stage clean (dbinfo only) -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - 'exposure' AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - NULL, - NULL, - NULL as state, - NULL as data_state, - 0 as quality, - distRun.no_magic, - rawExp.magicked -FROM distRun -JOIN rawExp ON exp_id = stage_id -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id -WHERE - distRun.state = 'new' - AND distRun.stage = 'raw' - AND distRun.clean - AND distComponent.dist_id IS NULL - -- where hook 2 %s - --- chip stage -UNION -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - chipProcessedImfile.class_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - chipProcessedImfile.path_base, - chipProcessedImfile.path_base as chip_path_base, - chipRun.state, - chipProcessedImfile.data_state, - chipProcessedImfile.quality, - distRun.no_magic, - chipProcessedImfile.magicked -FROM distRun -JOIN chipRun ON chipRun.chip_id = distRun.stage_id -JOIN rawExp using(exp_id) -JOIN chipProcessedImfile ON chipProcessedImfile.chip_id = stage_id -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id - AND chipProcessedImfile.class_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.stage = 'chip' - AND distComponent.dist_id IS NULL - AND (distRun.clean OR chipRun.magicked OR distRun.no_magic) - AND (chipRun.state = 'full' OR (distRun.clean AND chipRun.state = 'cleaned')) - -- where hook 3 %s -UNION -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - 'exposure' AS component, --- chipProcessedImfile.class_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - camProcessedExp.path_base, - NULL as chip_path_base, - camRun.state, - NULL, - camProcessedExp.quality, - distRun.no_magic, - chipRun.magicked -FROM distRun -JOIN camRun ON camRun.cam_id = distRun.stage_id -JOIN camProcessedExp USING(cam_id) -JOIN chipRun USING(chip_id) --- JOIN chipProcessedImfile USING(exp_id, chip_id) -JOIN rawExp using(exp_id) -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id --- AND chipProcessedImfile.class_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.stage = 'camera' - AND distComponent.dist_id IS NULL - AND (distRun.clean OR (chipRun.magicked AND camRun.magicked) OR distRun.no_magic) - AND (camRun.state = 'full' OR (distRun.clean AND camRun.state = 'cleaned')) - -- where hook 4 %s -UNION -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - fakeProcessedImfile.class_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - fakeProcessedImfile.path_base, - NULL, - fakeRun.state, - NULL, - 0 as quality, - distRun.no_magic, - 0 -FROM distRun -JOIN fakeRun ON fakeRun.fake_id = distRun.stage_id -JOIN fakeProcessedImfile USING(fake_id) -JOIN camRun USING(cam_id) -JOIN chipRun USING(chip_id, exp_id) -JOIN rawExp using(exp_id) -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id - AND fakeProcessedImfile.class_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.stage = 'fake' - AND distComponent.dist_id IS NULL - -- where hook 5 %s -UNION -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - warpSkyfile.skycell_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - warpSkyfile.path_base, - NULL as chip_path_base, - warpRun.state, - warpSkyfile.data_state, - warpSkyfile.quality, - distRun.no_magic, - warpSkyfile.magicked -FROM distRun -JOIN warpRun ON stage_id = warp_id -JOIN warpSkyfile using(warp_id) -JOIN fakeRun USING(fake_id) -JOIN camRun USING(cam_id) -JOIN chipRun ON camRun.chip_id = chipRun.chip_id -JOIN rawExp using(exp_id) -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id - AND warpSkyfile.skycell_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.stage = 'warp' - AND distComponent.dist_id IS NULL - AND (distRun.clean OR warpRun.magicked OR distRun.no_magic) - AND (warpRun.state = 'full' OR (distRun.clean AND warpRun.state = 'cleaned')) - -- where hook 6 %s -UNION -SELECT - distRun.dist_id, - distRun.label, - stage, - stage_id, - diffSkyfile.skycell_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - diffSkyfile.path_base, - NULL as chip_path_base, - diffRun.state, - -- data_state doesn't exist yet - -- diffSkyfile.data_state, - 'full' AS data_state, - diffSkyfile.quality, - distRun.no_magic, - diffSkyfile.magicked -FROM distRun -JOIN diffRun ON stage_id = diff_id -JOIN diffSkyfile using(diff_id) -JOIN diffInputSkyfile USING(diff_id, skycell_id) -JOIN warpRun - ON warpRun.warp_id = diffInputSkyfile.warp1 -JOIN fakeRun USING(fake_id) -JOIN camRun USING(cam_id) -JOIN chipRun USING(chip_id) -JOIN rawExp USING(exp_id) -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id - AND diffSkyfile.skycell_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.stage = 'diff' - AND distComponent.dist_id IS NULL - AND (distRun.clean OR diffRun.magicked OR distRun.no_magic) - AND (diffRun.state = 'full' OR (distRun.clean AND diffRun.state = 'cleaned')) - -- where hook 7 %s -UNION -SELECT DISTINCT - distRun.dist_id, - distRun.label, - stage, - stage_id, - stackRun.skycell_id AS component, - clean, - rawExp.camera, - CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, - stackSumSkyfile.path_base, - NULL as chip_path_base, - stackRun.state, - -- data_state doesn't exist yet - -- stackSumSkyfile.data_state, - 'full' AS data_state, - stackSumSkyfile.quality, - 1 AS no_magic, - 0 AS magicked -FROM distRun -JOIN stackRun - ON stage_id = stack_id -JOIN stackSumSkyfile - USING(stack_id) -JOIN stackInputSkyfile - USING(stack_id) -JOIN warpSkyfile - ON stackInputSkyfile.warp_id = warpSkyfile.warp_id - AND stackRun.skycell_id = warpSkyfile.skycell_id - AND stackRun.tess_id = warpSkyfile.tess_id -JOIN warpRun - ON warpRun.warp_id = warpSkyfile.warp_id -JOIN fakeRun - USING(fake_id) -JOIN camRun - USING(cam_id) -JOIN chipRun - ON camRun.chip_id = chipRun.chip_id -JOIN rawExp - USING (exp_id) -LEFT JOIN distComponent - ON distRun.dist_id = distComponent.dist_id - AND stackRun.skycell_id = distComponent.component -WHERE - distRun.state = 'new' - AND distRun.stage = 'stack' - AND distComponent.dist_id IS NULL - AND (stackRun.state = 'full' OR (distRun.clean AND stackRun.state = 'cleaned')) - -- where hook 8 %s -) as Foo Index: branches/eam_branches/20090820/ippTools/share/disttool_pendingfileset.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_pendingfileset.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/disttool_pendingfileset.sql (revision 25766) @@ -4,15 +4,16 @@ distRun.stage, stage_id, + distTarget.label, + distTarget.filter, CONCAT_WS('.', distRun.outroot, CONVERT(distRun.dist_id, CHAR)) as dist_dir, - rcDSProduct.name AS product_name, - rcDSProduct.prod_id, - rcDSProduct.dbname AS ds_dbname, - rcDSProduct.dbhost AS ds_dbhost + rcDestination.name AS product_name, + rcDestination.dest_id, + rcDestination.dbname AS ds_dbname, + rcDestination.dbhost AS ds_dbhost FROM rcDestination JOIN rcInterest USING(dest_id) JOIN distTarget USING(target_id) -JOIN rcDSProduct USING(prod_id) JOIN distRun USING(target_id) -LEFT JOIN rcDSFileset USING(prod_id, dist_id) +LEFT JOIN rcDSFileset USING(dest_id, dist_id) WHERE distRun.state = 'full' AND rcDestination.state = 'enabled' Index: branches/eam_branches/20090820/ippTools/share/disttool_queuercrun.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_queuercrun.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/disttool_queuercrun.sql (revision 25766) @@ -13,7 +13,6 @@ JOIN rcInterest USING(dest_id) JOIN distTarget USING(target_id) -JOIN rcDSProduct USING(prod_id) JOIN distRun USING(target_id, stage) -JOIN rcDSFileset USING(prod_id, dist_id) +JOIN rcDSFileset USING(dest_id, dist_id) LEFT JOIN rcRun using(fs_id, dest_id) WHERE rcRun.rc_id IS NULL Index: branches/eam_branches/20090820/ippTools/share/disttool_revertcomponent.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_revertcomponent.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/disttool_revertcomponent.sql (revision 25766) @@ -0,0 +1,5 @@ +DELETE FROM distComponent +USING distComponent, distRun +WHERE distComponent.dist_id = distRun.dist_id + AND distRun.state = 'new' + AND distComponent.fault != 0 Index: branches/eam_branches/20090820/ippTools/share/disttool_revertrun.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_revertrun.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/disttool_revertrun.sql (revision 25766) @@ -0,0 +1,4 @@ +UPDATE distRun +SET distRun.fault = 0 +WHERE distRun.state = 'new' + AND distRun.fault != 0 Index: branches/eam_branches/20090820/ippTools/share/disttool_revertrun_delete.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_revertrun_delete.sql (revision 25206) +++ (revision ) @@ -1,5 +1,0 @@ -DELETE FROM distComponent -USING distComponent, distRun -WHERE distComponent.dist_id = distRun.dist_id - AND distRun.state = 'new' - AND distComponent.fault != 0 Index: branches/eam_branches/20090820/ippTools/share/disttool_revertrun_update.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_revertrun_update.sql (revision 25206) +++ (revision ) @@ -1,4 +1,0 @@ -UPDATE distRun -SET distRun.fault = 0 -WHERE distRun.state = 'new' - AND distRun.fault != 0 Index: branches/eam_branches/20090820/ippTools/share/disttool_updatercrun.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/disttool_updatercrun.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/disttool_updatercrun.sql (revision 25766) @@ -1,6 +1,5 @@ UPDATE rcRun JOIN rcDestination USING(dest_id) -JOIN rcDSProduct USING(prod_id) -JOIN rcDSFileset using(prod_id, fs_id) +JOIN rcDSFileset using(dest_id, fs_id) SET -- set hook %s Index: branches/eam_branches/20090820/ippTools/share/faketool_pendingcleanupimfile.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/faketool_pendingcleanupimfile.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/faketool_pendingcleanupimfile.sql (revision 25766) @@ -13,3 +13,7 @@ USING(fake_id) WHERE - fakeRun.state = 'goto_cleaned' + ((fakeRun.state = 'goto_cleaned' AND fakeProcessedImfile.data_state = 'full') +OR + (fakeRun.state = 'goto_scrubbed' AND fakeProcessedImfile.data_state = 'full') +OR + (fakeRun.state = 'goto_purged' AND fakeProcessedImfile.data_state != 'purged')) Index: branches/eam_branches/20090820/ippTools/share/faketool_pendingcleanuprun.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/faketool_pendingcleanuprun.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/faketool_pendingcleanuprun.sql (revision 25766) @@ -11,3 +11,4 @@ USING (exp_id) WHERE - fakeRun.state = 'goto_cleaned' + (fakeRun.state = 'goto_cleaned' OR fakeRun.state = 'goto_scrubbed' OR fakeRun.state = 'goto_purged') + Index: branches/eam_branches/20090820/ippTools/share/magicdstool_completed_runs.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magicdstool_completed_runs.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/magicdstool_completed_runs.sql (revision 25766) @@ -1,9 +1,11 @@ SELECT DISTINCT - magic_ds_id + magic_ds_id, + re_place, + label FROM ( -- raw stage SELECT - magicDSRun.magic_ds_id + magicDSRun.* FROM magicDSRun JOIN rawImfile ON stage_id = rawImfile.exp_id @@ -23,5 +25,5 @@ -- chip stage SELECT - magicDSRun.magic_ds_id + magicDSRun.* FROM magicDSRun JOIN chipProcessedImfile ON stage_id = chip_id @@ -42,5 +44,5 @@ -- camera stage SELECT - magicDSRun.magic_ds_id + magicDSRun.* FROM magicDSRun JOIN camProcessedExp ON stage_id = camProcessedExp.cam_id @@ -59,5 +61,5 @@ -- warp stage SELECT - magicDSRun.magic_ds_id + magicDSRun.* FROM magicDSRun JOIN warpSkyfile on stage_id = warp_id @@ -79,5 +81,5 @@ -- diff stage SELECT DISTINCT - magicDSRun.magic_ds_id + magicDSRun.* FROM magicDSRun JOIN magicRun USING (magic_id) @@ -101,3 +103,3 @@ AND SUM(magicDSFile.fault) = 0 - ) as Foo + ) as magicDSRun Index: branches/eam_branches/20090820/ippTools/share/magicdstool_completedrevert.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magicdstool_completedrevert.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/magicdstool_completedrevert.sql (revision 25766) @@ -0,0 +1,10 @@ +SELECT + magic_ds_id, + state +FROM magicDSRun +LEFT JOIN magicDSFile USING(magic_ds_id) +WHERE + magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored' +GROUP BY magic_ds_id +HAVING COUNT(component) = 0 + Index: branches/eam_branches/20090820/ippTools/share/magicdstool_revertdestreakedfile.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magicdstool_revertdestreakedfile.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/magicdstool_revertdestreakedfile.sql (revision 25766) @@ -0,0 +1,7 @@ +DELETE FROM magicDSFile USING magicDSFile, magicDSRun +WHERE (magicDSRun.magic_ds_id = magicDSFile.magic_ds_id) + AND ((magicDSRun.state = 'new' AND magicDSFile.fault != 0) + OR (magicDSRun.state = 'goto_restored' + OR magicDSRun.state = 'goto_censored' + ) + ) Index: branches/eam_branches/20090820/ippTools/share/magicdstool_todestreak.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magicdstool_todestreak.sql (revision 25206) +++ (revision ) @@ -1,228 +1,0 @@ -SELECT * -FROM ( -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicRun.magic_id, - magicRun.exp_id, - magicDSRun.label, - camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - stage_id, - class_id as component, - rawImfile.uri as uri, - NULL as path_base, - magicRun.inverse, - camProcessedExp.path_base as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM magicDSRun -JOIN magicMask USING (magic_id) -JOIN magicRun USING (magic_id) -JOIN camProcessedExp USING(cam_id) -JOIN rawImfile ON magicRun.exp_id = rawImfile.exp_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = rawImfile.class_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'raw' - AND magicDSFile.component IS NULL -UNION - -- chipProcesedImfiles -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicDSRun.magic_id, - chipRun.exp_id, - magicDSRun.label, - camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - stage_id, - class_id as component, - chipProcessedImfile.uri, - chipProcessedImfile.path_base, - magicRun.inverse, - camProcessedExp.path_base as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM magicDSRun -JOIN magicMask USING (magic_id) -JOIN magicRun USING(magic_id) -JOIN camProcessedExp USING(cam_id) -JOIN chipRun ON chip_id = stage_id -JOIN chipProcessedImfile USING(chip_id) -JOIN rawExp ON chipRun.exp_id = rawExp.exp_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = chipProcessedImfile.class_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'chip' - AND chipRun.state = 'full' - AND chipProcessedImfile.fault = 0 - AND chipProcessedImfile.quality = 0 - AND magicDSFile.component IS NULL -UNION - -- camProcessedExp -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicDSRun.magic_id, - chipRun.exp_id, - magicDSRun.label, - camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - stage_id, - 'exposure' as component, - NULL AS uri, - camProcessedExp.path_base, - magicRun.inverse, - camProcessedExp.path_base as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM magicDSRun -JOIN magicMask USING (magic_id) -JOIN magicRun USING(magic_id) -JOIN camRun ON magicDSRun.stage_id = camRun.cam_id -JOIN camProcessedExp ON camRun.cam_id = camProcessedExp.cam_id -JOIN chipRun USING(chip_id) -JOIN rawExp ON chipRun.exp_id = rawExp.exp_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'camera' - AND camRun.state = 'full' - AND chipRun.state = 'full' - AND chipRun.magicked - AND camProcessedExp.fault = 0 - AND camProcessedExp.quality = 0 - AND magicDSFile.component IS NULL -UNION --- warpSkyfiles -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicRun.magic_id, - magicRun.exp_id, - magicDSRun.label, - camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - stage_id, - warpSkyfile.skycell_id as component, - warpSkyfile.uri, - warpSkyfile.path_base, - magicRun.inverse, - NULL as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM magicDSRun -JOIN magicMask USING (magic_id) -JOIN magicRun USING (magic_id) -JOIN warpRun ON warp_id = stage_id -JOIN warpSkyfile USING(warp_id) -JOIN rawExp ON magicRun.exp_id = rawExp.exp_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = warpSkyfile.skycell_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'warp' - AND warpRun.state = 'full' - AND warpSkyfile.fault = 0 - AND warpSkyfile.quality = 0 - AND magicDSFile.component IS NULL -UNION --- regular diffSkyfiles -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicRun.magic_id, - magicRun.exp_id, - magicDSRun.label, - rawExp.camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - magicRun.diff_id as stage_id, - diffSkyfile.skycell_id as component, - NULL AS uri, - diffSkyfile.path_base, - magicRun.inverse, - NULL as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM rawExp -JOIN magicRun USING (exp_id) -JOIN magicMask USING (magic_id) -JOIN magicDSRun USING(magic_id) -JOIN magicInputSkyfile USING(magic_id) -JOIN diffRun USING(diff_id) -JOIN diffSkyfile - ON magicRun.diff_id = diffSkyfile.diff_id - AND magicInputSkyfile.node = diffSkyfile.skycell_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = diffSkyfile.skycell_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'diff' - AND diffRun.bothways = 0 - AND diffSkyfile.fault = 0 - AND diffSkyfile.quality = 0 - AND magicDSFile.component IS NULL --- bothways diffSkyfiles -UNION -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicRun.magic_id, - magicRun.exp_id, - magicDSRun.label, - rawExp.camera, - magicMask.uri as streaks_uri, - (SELECT uri from magicMask where magic_id = inv_magic_id) AS inv_streaks_uri, - stage, - magicRun.diff_id as stage_id, - diffSkyfile.skycell_id as component, - NULL AS uri, - diffSkyfile.path_base, - magicRun.inverse, - NULL as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM rawExp -JOIN magicRun USING (exp_id) -JOIN magicMask USING (magic_id) -JOIN magicDSRun USING(magic_id) -JOIN magicInputSkyfile USING(magic_id) -JOIN diffRun USING(diff_id) -JOIN diffSkyfile - ON magicRun.diff_id = diffSkyfile.diff_id - AND magicInputSkyfile.node = diffSkyfile.skycell_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = diffSkyfile.skycell_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'diff' - AND diffRun.bothways - AND diffSkyfile.fault = 0 - AND diffSkyfile.quality = 0 - AND magicDSFile.component IS NULL -) as Foo Index: branches/eam_branches/20090820/ippTools/share/magicdstool_todestreak_camera.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magicdstool_todestreak_camera.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/magicdstool_todestreak_camera.sql (revision 25766) @@ -0,0 +1,37 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicDSRun.magic_id, + chipRun.exp_id, + magicDSRun.label, + camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + stage_id, + 'exposure' as component, + CAST(NULL AS CHAR(255)) AS uri, + camProcessedExp.path_base, + magicRun.inverse, + camProcessedExp.path_base as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING(magic_id) +JOIN camRun ON magicDSRun.stage_id = camRun.cam_id +JOIN camProcessedExp ON camRun.cam_id = camProcessedExp.cam_id +JOIN chipRun USING(chip_id) +JOIN rawExp ON chipRun.exp_id = rawExp.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'camera' + AND camRun.state = 'full' + AND chipRun.state = 'full' + AND chipRun.magicked + AND camProcessedExp.fault = 0 + AND camProcessedExp.quality = 0 + AND magicDSFile.component IS NULL Index: branches/eam_branches/20090820/ippTools/share/magicdstool_todestreak_chip.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magicdstool_todestreak_chip.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/magicdstool_todestreak_chip.sql (revision 25766) @@ -0,0 +1,36 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicDSRun.magic_id, + chipRun.exp_id, + magicDSRun.label, + camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + stage_id, + class_id as component, + chipProcessedImfile.uri, + chipProcessedImfile.path_base, + magicRun.inverse, + camProcessedExp.path_base as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING(magic_id) +JOIN camProcessedExp USING(cam_id) +JOIN chipRun ON chip_id = stage_id +JOIN chipProcessedImfile USING(chip_id) +JOIN rawExp ON chipRun.exp_id = rawExp.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = chipProcessedImfile.class_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'chip' + AND chipRun.state = 'full' + AND chipProcessedImfile.fault = 0 + AND chipProcessedImfile.quality = 0 + AND magicDSFile.component IS NULL Index: branches/eam_branches/20090820/ippTools/share/magicdstool_todestreak_diff.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magicdstool_todestreak_diff.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/magicdstool_todestreak_diff.sql (revision 25766) @@ -0,0 +1,82 @@ +SELECT * FROM ( +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicRun.exp_id, + magicDSRun.label, + rawExp.camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + magicRun.diff_id as stage_id, + diffSkyfile.skycell_id as component, + CAST(NULL AS CHAR(255)) AS uri, + diffSkyfile.path_base, + magicRun.inverse, + CAST(NULL AS CHAR(255)) as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM rawExp +JOIN magicRun USING (exp_id) +JOIN magicMask USING (magic_id) +JOIN magicDSRun USING(magic_id) +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffSkyfile + ON magicRun.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = diffSkyfile.skycell_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'diff' + AND diffRun.bothways = 0 + AND diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 + AND magicDSFile.component IS NULL +-- bothways diffSkyfiles +UNION +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicRun.exp_id, + magicDSRun.label, + rawExp.camera, + magicMask.uri as streaks_uri, + (SELECT uri from magicMask where magic_id = inv_magic_id) AS inv_streaks_uri, + stage, + magicRun.diff_id as stage_id, + diffSkyfile.skycell_id as component, + CAST(NULL AS CHAR(255)) AS uri, + diffSkyfile.path_base, + magicRun.inverse, + CAST(NULL AS CHAR(255)) as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM rawExp +JOIN magicRun USING (exp_id) +JOIN magicMask USING (magic_id) +JOIN magicDSRun USING(magic_id) +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffSkyfile + ON magicRun.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = diffSkyfile.skycell_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'diff' + AND diffRun.bothways + AND diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 + AND magicDSFile.component IS NULL +) as magicDSRun +-- we need the following so this query is compatible with the other stages +WHERE magic_ds_id IS NOT NULL Index: branches/eam_branches/20090820/ippTools/share/magicdstool_todestreak_raw.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magicdstool_todestreak_raw.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/magicdstool_todestreak_raw.sql (revision 25766) @@ -0,0 +1,31 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicRun.exp_id, + magicDSRun.label, + camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + stage_id, + class_id as component, + rawImfile.uri as uri, + CAST(NULL AS CHAR(255)) as path_base, + magicRun.inverse, + camProcessedExp.path_base as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING (magic_id) +JOIN camProcessedExp USING(cam_id) +JOIN rawImfile ON magicRun.exp_id = rawImfile.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = rawImfile.class_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'raw' + AND magicDSFile.component IS NULL Index: branches/eam_branches/20090820/ippTools/share/magicdstool_todestreak_warp.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magicdstool_todestreak_warp.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/magicdstool_todestreak_warp.sql (revision 25766) @@ -0,0 +1,35 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicRun.exp_id, + magicDSRun.label, + camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + stage_id, + warpSkyfile.skycell_id as component, + warpSkyfile.uri, + warpSkyfile.path_base, + magicRun.inverse, + CAST(NULL AS CHAR(255)) as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING (magic_id) +JOIN warpRun ON warp_id = stage_id +JOIN warpSkyfile USING(warp_id) +JOIN rawExp ON magicRun.exp_id = rawExp.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = warpSkyfile.skycell_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'warp' + AND warpRun.state = 'full' + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 + AND magicDSFile.component IS NULL Index: branches/eam_branches/20090820/ippTools/share/magicdstool_torestore.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magicdstool_torestore.sql (revision 25206) +++ (revision ) @@ -1,12 +1,0 @@ -SELECT - magic_ds_id, - stage, - stage_id - component, - backup_path_base, - recovery_path_base -FROM magicDSFile -JOIN magicDSRun USING(magic_ds_id) -WHERE magicDSRun.state = 'goto_restored' - AND backup_path_base IS NOT NULL - AND recovery_path_base IS NOT NULL Index: branches/eam_branches/20090820/ippTools/share/magicdstool_torevert_camera.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magicdstool_torevert_camera.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/magicdstool_torevert_camera.sql (revision 25766) @@ -1,4 +1,5 @@ SELECT magic_ds_id, + magicDSRun.state, exp_id, re_place, @@ -19,4 +20,7 @@ JOIN rawExp using(exp_id) WHERE magicDSRun.stage = 'camera' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR ((magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored') + AND (backup_path_base IS NOT NULL) + ) + ) Index: branches/eam_branches/20090820/ippTools/share/magicdstool_torevert_chip.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magicdstool_torevert_chip.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/magicdstool_torevert_chip.sql (revision 25766) @@ -1,4 +1,5 @@ SELECT magic_ds_id, + magicDSRun.state, exp_id, re_place, @@ -18,4 +19,7 @@ JOIN rawExp using(exp_id) WHERE magicDSRun.stage = 'chip' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR ((magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored') + AND ((backup_path_base IS NOT NULL) OR (recovery_path_base IS NOT NULL)) + ) + ) Index: branches/eam_branches/20090820/ippTools/share/magicdstool_torevert_diff.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magicdstool_torevert_diff.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/magicdstool_torevert_diff.sql (revision 25766) @@ -1,4 +1,5 @@ SELECT magic_ds_id, + magicDSRun.state, exp_id, re_place, @@ -18,4 +19,7 @@ JOIN rawExp USING(exp_id) WHERE magicDSRun.stage = 'diff' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR ((magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored') + AND ((backup_path_base IS NOT NULL) OR (recovery_path_base IS NOT NULL)) + ) + ) Index: branches/eam_branches/20090820/ippTools/share/magicdstool_torevert_raw.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magicdstool_torevert_raw.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/magicdstool_torevert_raw.sql (revision 25766) @@ -1,4 +1,5 @@ SELECT magic_ds_id, + magicDSRun.state, exp_id, re_place, @@ -15,7 +16,8 @@ JOIN magicDSFile using(magic_ds_id) JOIN rawImfile ON (stage_id = exp_id AND component = class_id) - JOIN summitImfile ON (rawImfile.exp_name = summitImfile.exp_name - AND rawImfile.tmp_class_id = summitImfile.class_id) WHERE magicDSRun.stage = 'raw' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR ((magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored') + AND ((backup_path_base IS NOT NULL) OR (recovery_path_base IS NOT NULL)) + ) + ) Index: branches/eam_branches/20090820/ippTools/share/magicdstool_torevert_warp.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magicdstool_torevert_warp.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/magicdstool_torevert_warp.sql (revision 25766) @@ -1,4 +1,5 @@ SELECT magic_ds_id, + magicDSRun.state, exp_id, re_place, @@ -21,4 +22,7 @@ JOIN rawExp USING(exp_id) WHERE magicDSRun.stage = 'warp' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR ((magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored') + AND ((backup_path_base IS NOT NULL) OR (recovery_path_base IS NOT NULL)) + ) + ) Index: branches/eam_branches/20090820/ippTools/share/magictool_censor_camera.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magictool_censor_camera.sql (revision 25206) +++ (revision ) @@ -1,8 +1,0 @@ -UPDATE magicRun - JOIN magicDSRun USING(magic_id) - JOIN magicDSFile USING(magic_ds_id) - JOIN camRun ON magic_ds_id = magicked -SET camRun.magicked = 0, - magicDSRun.state = 'censored', - magicDSFile.fault = 42 - Index: branches/eam_branches/20090820/ippTools/share/magictool_censor_chip.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magictool_censor_chip.sql (revision 25206) +++ (revision ) @@ -1,9 +1,0 @@ -UPDATE chipRun - JOIN chipProcessedImfile USING(chip_id, exp_id) - JOIN magicDSRun ON chipRun.magicked = magicDSRun.magic_ds_id - JOIN magicDSFile USING(magic_ds_id) - JOIN magicRun USING(magic_id, exp_id) -SET chipProcessedImfile.magicked = 0, - chipRun.magicked = 0, - magicDSRun.state = 'censored', - magicDSFile.fault = 42 Index: branches/eam_branches/20090820/ippTools/share/magictool_censor_diff.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magictool_censor_diff.sql (revision 25206) +++ (revision ) @@ -1,9 +1,0 @@ -UPDATE magicRun - JOIN magicDSRun USING(magic_id) - JOIN magicDSFile USING(magic_ds_id) - JOIN diffRun ON magic_ds_id = magicked - JOIN diffSkyfile ON diffRun.diff_id = diffSkyfile.diff_id -SET diffRun.magicked = 0, - diffSkyfile.magicked = 0, - magicDSRun.state = 'censored', - magicDSFile.fault = 42 Index: branches/eam_branches/20090820/ippTools/share/magictool_censor_raw.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magictool_censor_raw.sql (revision 25206) +++ (revision ) @@ -1,10 +1,0 @@ -UPDATE magicRun - JOIN magicDSRun USING(magic_id) - JOIN magicDSFile USING(magic_ds_id) - JOIN rawExp ON rawExp.magicked = magic_ds_id - JOIN rawImfile ON rawExp.exp_id = rawImfile.exp_id -SET rawImfile.magicked = 0, - rawExp.magicked = 0, - magicDSRun.state = 'censored', - magicDSFile.fault = 42 - Index: branches/eam_branches/20090820/ippTools/share/magictool_censor_warp.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magictool_censor_warp.sql (revision 25206) +++ (revision ) @@ -1,9 +1,0 @@ -UPDATE magicRun - JOIN magicDSRun USING(magic_id) - JOIN magicDSFile USING(magic_ds_id) - JOIN warpRun ON magic_ds_id = magicked - JOIN warpSkyfile using(warp_id) -SET warpRun.magicked = 0, - warpSkyfile.magicked = 0, - magicDSRun.state = 'censored', - magicDSFile.fault = 42 Index: branches/eam_branches/20090820/ippTools/share/magictool_restore_camera.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magictool_restore_camera.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/magictool_restore_camera.sql (revision 25766) @@ -0,0 +1,6 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN camRun ON stage = 'camera' AND stage_id = camRun.cam_id +SET camRun.magicked = 0, + magicDSRun.state = '@NEW_STATE@' +WHERE re_place Index: branches/eam_branches/20090820/ippTools/share/magictool_restore_chip.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magictool_restore_chip.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/magictool_restore_chip.sql (revision 25766) @@ -0,0 +1,8 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN chipRun ON stage = 'chip' AND stage_id = chip_id + JOIN chipProcessedImfile USING(chip_id) +SET chipProcessedImfile.magicked = 0, + chipRun.magicked = 0, + magicDSRun.state = '@NEW_STATE@' +WHERE re_place Index: branches/eam_branches/20090820/ippTools/share/magictool_restore_diff.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magictool_restore_diff.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/magictool_restore_diff.sql (revision 25766) @@ -0,0 +1,8 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN diffRun ON stage = 'diff' AND stage_id = diffRun.diff_id + JOIN diffSkyfile ON diffRun.diff_id = diffSkyfile.diff_id +SET diffRun.magicked = 0, + diffSkyfile.magicked = 0, + magicDSRun.state = '@NEW_STATE@' +WHERE re_place Index: branches/eam_branches/20090820/ippTools/share/magictool_restore_raw.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magictool_restore_raw.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/magictool_restore_raw.sql (revision 25766) @@ -0,0 +1,8 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN rawExp ON stage = 'raw' AND stage_id = rawExp.exp_id + JOIN rawImfile ON rawExp.exp_id = rawImfile.exp_id +SET rawImfile.magicked = 0, + rawExp.magicked = 0, + magicDSRun.state = '@NEW_STATE@' +WHERE re_place Index: branches/eam_branches/20090820/ippTools/share/magictool_restore_warp.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/magictool_restore_warp.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/magictool_restore_warp.sql (revision 25766) @@ -0,0 +1,8 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN warpRun ON stage = 'warp' AND stage_id = warp_id + JOIN warpSkyfile using(warp_id) +SET warpRun.magicked = 0, + warpSkyfile.magicked = 0, + magicDSRun.state = '@NEW_STATE@' +WHERE re_place Index: branches/eam_branches/20090820/ippTools/share/pstamptool_completedreq.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/pstamptool_completedreq.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/pstamptool_completedreq.sql (revision 25766) @@ -0,0 +1,8 @@ +SELECT * +FROM pstampRequest +WHERE state = 'run' + AND ( + SELECT count(*) FROM pstampJob + WHERE pstampJob.req_id = pstampRequest.req_id + AND pstampJob.state != 'stop' + ) = 0 Index: branches/eam_branches/20090820/ippTools/share/pstamptool_listjob.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/pstamptool_listjob.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/pstamptool_listjob.sql (revision 25766) @@ -0,0 +1,6 @@ +SELECT + pstampJob.*, + pstampRequest.name, + pstampRequest.outProduct +FROM pstampJob + JOIN pstampRequest USING(req_id) Index: branches/eam_branches/20090820/ippTools/share/pstamptool_pendingjob.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/pstamptool_pendingjob.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/pstamptool_pendingjob.sql (revision 25766) @@ -1,4 +1,7 @@ -SELECT * - FROM pstampJob - WHERE state = 'run' - AND fault = 0 +SELECT pstampJob.* +FROM pstampJob + JOIN pstampRequest using(req_id) +WHERE pstampRequest.state = 'run' + AND pstampRequest.fault = 0 + AND pstampJob.state = 'run' + AND pstampJob.fault = 0 Index: branches/eam_branches/20090820/ippTools/share/pstamptool_pendingreq.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/pstamptool_pendingreq.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/pstamptool_pendingreq.sql (revision 25766) @@ -7,2 +7,3 @@ USING(ds_id) WHERE pstampRequest.state = 'new' + AND pstampRequest.fault = 0 Index: branches/eam_branches/20090820/ippTools/share/pstamptool_revertjob.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/pstamptool_revertjob.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/pstamptool_revertjob.sql (revision 25766) @@ -0,0 +1,7 @@ +UPDATE pstampJob + JOIN pstampRequest USING(req_id) +SET pstampJob.fault = 0 +WHERE pstampRequest.state = 'run' + AND pstampJob.state = 'run' + # do not revert faults >= 10 those are faults due to requestor errors + AND pstampJob.fault < 10 Index: branches/eam_branches/20090820/ippTools/share/pstamptool_revertreq.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/pstamptool_revertreq.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/pstamptool_revertreq.sql (revision 25766) @@ -0,0 +1,4 @@ +UPDATE pstampRequest + SET fault = 0 +WHERE pstampRequest.state != 'stop' + Index: branches/eam_branches/20090820/ippTools/share/pstamptool_revertreq_deletejobs.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/pstamptool_revertreq_deletejobs.sql (revision 25766) +++ branches/eam_branches/20090820/ippTools/share/pstamptool_revertreq_deletejobs.sql (revision 25766) @@ -0,0 +1,5 @@ +-- delete jobs that got added by an incomplete pstamp parser run +DELETE pstampJob +FROM pstampJob + JOIN pstampRequest USING(req_id) +WHERE pstampRequest.state = 'new' Index: branches/eam_branches/20090820/ippTools/share/pxadmin_create_tables.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/pxadmin_create_tables.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/pxadmin_create_tables.sql (revision 25766) @@ -58,4 +58,6 @@ epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, hostname VARCHAR(64), + bytes INT, + md5sum VARCHAR(32), PRIMARY KEY(exp_name, camera, telescope, class, class_id), KEY(fault), @@ -94,4 +96,6 @@ uri VARCHAR(255), epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + bytes INT, + md5sum VARCHAR(32), PRIMARY KEY(exp_id, tmp_class_id), FOREIGN KEY(exp_id) REFERENCES newExp(exp_id) @@ -237,4 +241,7 @@ raw_image_id BIGINT AUTO_INCREMENT, magicked BIGINT, + bytes INT, + md5sum VARCHAR(32), + burntool_state SMALLINT, PRIMARY KEY(exp_id, class_id), KEY(tmp_class_id), @@ -451,4 +458,38 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE addRun ( + add_id BIGINT AUTO_INCREMENT, + cam_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + dvodb VARCHAR(255), + magicked BIGINT, + PRIMARY KEY(add_id), + KEY(add_id), + KEY(cam_id), + KEY(state), + KEY(workdir_state), + KEY(label), + INDEX(add_id, cam_id), + FOREIGN KEY(cam_id) REFERENCES camRun(cam_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE addProcessedExp ( + add_id BIGINT AUTO_INCREMENT, + dtime_addstar FLOAT, + n_stars INT, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(add_id), + FOREIGN KEY(add_id) REFERENCES addRun(add_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE addMask ( + label VARCHAR(64), + PRIMARY KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE fakeRun ( fake_id BIGINT AUTO_INCREMENT, @@ -1191,4 +1232,5 @@ name VARCHAR(64) UNIQUE, reqType VARCHAR(16), + label VARCHAR(64), outProduct VARCHAR(64), uri VARCHAR(255), @@ -1207,4 +1249,5 @@ exp_id BIGINT, outputBase VARCHAR(255), + options BIGINT, PRIMARY KEY(job_id, req_id), KEY(job_id), @@ -1229,4 +1272,5 @@ stage VARCHAR(64), stage_id BIGINT, + magic_ds_id BIGINT, label VARCHAR(64), outroot VARCHAR(255), @@ -1256,24 +1300,15 @@ -CREATE TABLE rcDSProduct ( - prod_id BIGINT AUTO_INCREMENT, - name VARCHAR(64), - dbname VARCHAR(64), - dbhost VARCHAR(64), - PRIMARY KEY(prod_id) -) ENGINE=innodb DEFAULT CHARSET=latin1; - CREATE TABLE rcDestination ( dest_id BIGINT AUTO_INCREMENT, - prod_id BIGINT, name VARCHAR(64), status_uri VARCHAR(255), comment VARCHAR(255), last_fileset VARCHAR(255), + dbname VARCHAR(64), + dbhost VARCHAR(64), state VARCHAR(64), - PRIMARY KEY(dest_id), - FOREIGN KEY(prod_id) REFERENCES rcDSProduct(prod_id) + PRIMARY KEY(dest_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; - CREATE TABLE rcInterest ( @@ -1291,14 +1326,13 @@ fs_id BIGINT AUTO_INCREMENT, dist_id BIGINT, - prod_id BIGINT, + dest_id BIGINT, name VARCHAR(255), state VARCHAR(64), registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, fault SMALLINT DEFAULT 0, - PRIMARY KEY(dist_id, prod_id), + PRIMARY KEY(dist_id, dest_id), KEY(fs_id), FOREIGN KEY(dist_id) REFERENCES distRun(dist_id), - FOREIGN KEY(prod_id) REFERENCES rcDSProduct(prod_id) --- KEY(dist_id, prod_id) + FOREIGN KEY(dest_id) REFERENCES rcDestination(dest_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1407,4 +1441,6 @@ pub_id BIGINT AUTO_INCREMENT, -- link to publishRun path_base VARCHAR(255), -- base path of output + hostname VARCHAR(64), -- name of host + dtime_script FLOAT, -- run time for script fault SMALLINT NOT NULL DEFAULT 0, -- Fault code PRIMARY KEY(pub_id), Index: branches/eam_branches/20090820/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/pxadmin_drop_tables.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/pxadmin_drop_tables.sql (revision 25766) @@ -19,4 +19,7 @@ DROP TABLE IF EXISTS detRun; DROP TABLE IF EXISTS detInputExp; +DROP TABLE IF EXISTS addRun; +DROP TABLE IF EXISTS addProcessedExp; +DROP TABLE IF EXISTS addMask; DROP TABLE IF EXISTS fakeRun; DROP TABLE IF EXISTS fakeProcessedImfile; Index: branches/eam_branches/20090820/ippTools/share/rcserver_updatercrun.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/rcserver_updatercrun.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/rcserver_updatercrun.sql (revision 25766) @@ -1,9 +1,4 @@ UPDATE rcRun JOIN rcDestination USING(dest_id) -JOIN rcDSProduct USING(prod_id) -JOIN rcDSFileset using(prod_id, fs_id) +JOIN rcDSFileset using(dest_id, fs_id) SET -rcRun.fault= 42, rcDestination.last_fileset = 'o4741g0236o.chip.14519.33.1' - - - Index: branches/eam_branches/20090820/ippTools/share/regtool_pendingimfile.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/regtool_pendingimfile.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/regtool_pendingimfile.sql (revision 25766) @@ -6,5 +6,7 @@ newExp.workdir, newImfile.tmp_class_id, - newImfile.uri + newImfile.uri, + newImfile.bytes, + newImfile.md5sum FROM newImfile JOIN newExp Index: branches/eam_branches/20090820/ippTools/share/regtool_updateprocessedimfile.sql =================================================================== --- branches/eam_branches/20090820/ippTools/share/regtool_updateprocessedimfile.sql (revision 25206) +++ branches/eam_branches/20090820/ippTools/share/regtool_updateprocessedimfile.sql (revision 25766) @@ -1,4 +1,4 @@ UPDATE rawImfile - SET user_1 = %lf + SET burntool_state = %d WHERE rawImfile.exp_id = %lld Index: branches/eam_branches/20090820/ippTools/src/Makefile.am =================================================================== --- branches/eam_branches/20090820/ippTools/src/Makefile.am (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/Makefile.am (revision 25766) @@ -1,3 +1,4 @@ bin_PROGRAMS = \ + addtool \ caltool \ camtool \ @@ -23,9 +24,6 @@ pubtool - -bin_SCRIPTS = \ - fakemagic - pkginclude_HEADERS = \ + pxadd.h \ pxadmin.h \ pxcam.h \ @@ -34,4 +32,5 @@ pxdata.h \ pxfake.h \ + pxmagic.h \ pxregister.h \ pxtag.h \ @@ -42,4 +41,5 @@ noinst_HEADERS = \ + addtool.h \ caltool.h \ camtool.h \ @@ -68,4 +68,5 @@ libpxtools_la_LDFLAGS = -release $(PACKAGE_VERSION) libpxtools_la_SOURCES = \ + pxadd.c \ pxcam.c \ pxchip.c \ @@ -75,4 +76,5 @@ pxfake.c \ pxfault.c \ + pxmagic.c \ pxregister.c \ pxtag.c \ @@ -96,4 +98,10 @@ pstamptool.c \ pstamptoolConfig.c + +addtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +addtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +addtool_SOURCES = \ + addtool.c \ + addtoolConfig.c caltool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) Index: branches/eam_branches/20090820/ippTools/src/addtool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/addtool.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/addtool.c (revision 25766) @@ -0,0 +1,1116 @@ +/* + * addtool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxadd.h" +#include "addtool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool pendingexpMode(pxConfig *config); +static bool addprocessedexpMode(pxConfig *config); +static bool processedexpMode(pxConfig *config); +static bool revertprocessedexpMode(pxConfig *config); +static bool updateprocessedexpMode(pxConfig *config); +static bool blockMode(pxConfig *config); +static bool maskedMode(pxConfig *config); +static bool unblockMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupexpMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); +static bool exportrunMode(pxConfig *config); +static bool importrunMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = addtoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(ADDTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(ADDTOOL_MODE_PENDINGEXP, pendingexpMode); + MODECASE(ADDTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpMode); + MODECASE(ADDTOOL_MODE_PROCESSEDEXP, processedexpMode); + MODECASE(ADDTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); + MODECASE(ADDTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + MODECASE(ADDTOOL_MODE_BLOCK, blockMode); + MODECASE(ADDTOOL_MODE_MASKED, maskedMode); + MODECASE(ADDTOOL_MODE_UNBLOCK, unblockMode); + MODECASE(ADDTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); + MODECASE(ADDTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpMode); + MODECASE(ADDTOOL_MODE_DONECLEANUP, donecleanupMode); + MODECASE(ADDTOOL_MODE_EXPORTRUN, exportrunMode); + MODECASE(ADDTOOL_MODE_IMPORTRUN, importrunMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + if (!psListLength(where->list) && + !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + + // find the cam_id of all the exposures that we want to queue up. + psString query = pxDataGet("addtool_find_cam_id.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // would could do this "all in the database" if we didn't want the option + // of changing the label/reduction/expgroup/dvodb/etc. So we're pulling the + // data out so we have the option of changing these values or leaving the + // old values in place (i.e., passing the values through). + + + // loop over our list of addRun rows + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + addRunRow *row = addRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into addRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + row->cam_id, + workdir ? workdir : row->workdir, + label ? label : row->label, + "RECIPE", + dvodb ? dvodb : row->dvodb + + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue chip_id: %" PRId64, row->cam_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "addRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + if ((!state) && (!label)) { + psError(PXTOOLS_ERR_DATA, false, "parameters are required"); + psFree(where); + return false; + } + + if (state) { + // set addRun.state to state + if (!pxaddRunSetStateByQuery(config, where, state)) { + psFree(where); + return false; + } + } + + if (label) { + // set addRun.label to label + if (!pxaddRunSetLabelByQuery(config, where, label)) { + psFree(where); + return false; + } + } + + psFree(where); + + return true; +} + + +static bool pendingexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("addtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addPendingExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +/* static bool pendingimfileMode(pxConfig *config) */ +/* { */ +/* PS_ASSERT_PTR_NON_NULL(config, false); */ + +/* psMetadata *where = psMetadataAlloc(); */ +/* pxaddGetSearchArgs (config, where); */ +/* PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); */ +/* pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); */ +/* PXOPT_COPY_S64(config->args, where, "-chip_id", "addRun.chip_id", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-class_id", "addProcessedExp.class_id", "=="); */ + +/* PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); */ + +/* psString query = pxDataGet("addtool_find_pendingimfile.sql"); */ +/* if (!query) { */ +/* psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); */ +/* return false; */ +/* } */ + +/* // use psDBGenerateWhereSQL because the SQL yields an intermediate table */ +/* if (psListLength(where->list)) { */ +/* psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); */ +/* psStringAppend(&query, " AND %s", whereClause); */ +/* psFree(whereClause); */ +/* } */ +/* psFree(where); */ + +/* if (!p_psDBRunQuery(config->dbh, query)) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* psFree(query); */ +/* return false; */ +/* } */ +/* psFree(query); */ + +/* psArray *output = p_psDBFetchResult(config->dbh); */ +/* if (!output) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* return false; */ +/* } */ +/* if (!psArrayLength(output)) { */ +/* psTrace("addtool", PS_LOG_INFO, "no rows found"); */ +/* psFree(output); */ +/* return true; */ +/* } */ + +/* // negate simple so the default is true */ +/* if (!ippdbPrintMetadatas(stdout, output, "addProcessedExp", !simple)) { */ +/* psError(PS_ERR_UNKNOWN, false, "failed to print array"); */ +/* psFree(output); */ +/* return false; */ +/* } */ + +/* psFree(output); */ + +/* return true; */ +/* } */ + +static bool addprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(add_id, config->args, "-add_id", true, false); + + // optional + PXOPT_LOOKUP_F32(dtime_addstar, config->args, "-dtime_addstar", false, false); + + PXOPT_LOOKUP_S32(n_stars, config->args, "-n_stars", false, false); + + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + + // generate restrictions + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + + psString query = pxDataGet("addtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClaus = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClaus); + psFree(whereClaus); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + addRunRow *pendingRow = addRunObjectFromMetadata(output->data[0]); + psFree(output); + addProcessedExpRow *row = addProcessedExpRowAlloc( + pendingRow->add_id, + dtime_addstar, + n_stars, + path_base, + 0 + ); + + if (!addProcessedExpInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + psFree(pendingRow); + return false; + } + + // since there is only one exp per 'new' set addRun.state = 'full' + if (!pxaddRunSetState(config, row->add_id, "full", magicked)) { + psError(PS_ERR_UNKNOWN, false, "failed to change addRun.state for add_id: %" PRId64, row->add_id); + psFree(row); + psFree(pendingRow); + return false; + } + + // NULL for end_stage means go as far as possible + // EAM : skip here if fault != 0 + // Also, we can run fake even if tess_id is not defined +/* if (fault || (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "add"))) { */ +/* psFree(row); */ +/* psFree(pendingRow); */ +/* if (!psDBCommit(config->dbh)) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* return false; */ +/* } */ +/* return true; */ +/* } */ + psFree(row); + // else continue on... + +/* if (!pxfakeQueueByAddID(config, */ +/* pendingRow->add_id, */ +/* pendingRow->workdir, */ +/* pendingRow->label, */ +/* pendingRow->reduction, */ +/* pendingRow->expgroup, */ +/* pendingRow->dvodb, */ +/* pendingRow->tess_id, */ +/* pendingRow->end_stage */ +/* )) { */ +/* // rollback */ +/* if (!psDBRollback(config->dbh)) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* } */ +/* psError(PS_ERR_UNKNOWN, false, "failed to queue new fakeRun"); */ +/* psFree(pendingRow); */ +/* return false; */ +/* } */ + psFree(pendingRow); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + // generate restrictions + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + if (!psListLength(where->list) && + !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters (or -all) are required"); + return false; + } + + psString query = pxDataGet("addtool_find_processedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + // we either add AND (condition) or WHERE (condition): + if (where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " AND addProcessedExp.fault != 0"); + } + if (where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND addProcessedExp.fault = 0"); + } + if (!where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " WHERE addProcessedExp.fault != 0"); + } + if (!where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " WHERE addProcessedExp.fault = 0"); + } + psFree(where); + + // order by add_id so that the postage stamp parser can easliy find the 'latest' astrometry + psStringAppend(&query, " ORDER BY add_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); +/* PXOPT_COPY_S16(config->args, where, "-fault", "addProcessedExp.fault", "=="); */ + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = pxDataGet("addtool_reset_faulted_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + + { + psString query = pxDataGet("addtool_revertprocessedexp.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updateprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); +/* PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); */ + + if (!pxSetFaultCode(config->dbh, "addProcessedExp", where, fault)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + if (!addMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM addMask"); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + char *query = "DELETE FROM addMask WHERE label = '%s'"; + + if (!p_psDBRunQueryF(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + + psString query = pxDataGet("addtool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addPendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool pendingcleanupexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(add_id, config->args, "-add_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (add_id) { + PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); + } + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + psString query = pxDataGet("addtool_pendingcleanupexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addPendingCleanupExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("addtool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool exportrunMode(pxConfig *config) +{ + typedef struct ExportTable { + char tableName[80]; + char sqlFilename[80]; + } ExportTable; + + int numExportTables = 2; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-add_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + FILE *f = fopen (outfile, "w"); + if (f == NULL) { + psError(PS_ERR_UNKNOWN, false, "failed to open output file"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); + + ExportTable tables [] = { + {"addRun", "addtool_export_run.sql"}, + {"addProcessedExp", "addtool_export_processed_exp.sql"}, + }; + + for (int i=0; i < numExportTables; i++) { + psString query = pxDataGet(tables[i].sqlFilename); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, true, "no rows found"); + psFree(output); + return false; + } + + if (clean) { + if (!strcmp(tables[i].tableName, "addRun")) { + if (!pxSetStateCleaned("addRun", "state", output)) { + psFree(output); + psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s", tables[i].tableName); + return false; + } + } + } + + // we must write the export table in non-simple (true) format + if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + } + + fclose (f); + + return true; +} + +bool importrunMode(pxConfig *config) +{ + unsigned int nFail; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); + + psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); + + fprintf (stdout, "---- input ----\n"); + psMetadataPrint (stderr, input, 1); + + psMetadataItem *item = psMetadataLookup (input, "addRun"); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + psMetadataItem *entry = psListGet (item->data.list, 0); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + addRunRow *addRun = addRunObjectFromMetadata (entry->data.md); + addRunInsertObject (config->dbh, addRun); + + // fprintf (stdout, "---- add run ----\n"); + // psMetadataPrint (stderr, entry->data.md, 1); + + item = psMetadataLookup (input, "addProcessedExp"); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + for (int i = 0; i < item->data.list->n; i++) { + psMetadataItem *entry = psListGet (item->data.list, i); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + addProcessedExpRow *addProcessedExp = addProcessedExpObjectFromMetadata (entry->data.md); + addProcessedExpInsertObject (config->dbh, addProcessedExp); + + // fprintf (stdout, "---- row %d ----\n", i); + // psMetadataPrint (stderr, entry->data.md, 1); + } + + return true; +} Index: branches/eam_branches/20090820/ippTools/src/addtool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/addtool.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/addtool.h (revision 25766) @@ -0,0 +1,46 @@ +/* + * addtool.h + * + * Copyright (C) 2006 Joshua Hoblitt, Christopher Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef ADDTOOL_H +#define ADDTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + ADDTOOL_MODE_NONE = 0x0, + ADDTOOL_MODE_DEFINEBYQUERY, + ADDTOOL_MODE_UPDATERUN, + ADDTOOL_MODE_PENDINGEXP, + ADDTOOL_MODE_ADDPROCESSEDEXP, + ADDTOOL_MODE_PROCESSEDEXP, + ADDTOOL_MODE_REVERTPROCESSEDEXP, + ADDTOOL_MODE_UPDATEPROCESSEDEXP, + ADDTOOL_MODE_BLOCK, + ADDTOOL_MODE_MASKED, + ADDTOOL_MODE_UNBLOCK, + ADDTOOL_MODE_PENDINGCLEANUPRUN, + ADDTOOL_MODE_PENDINGCLEANUPEXP, + ADDTOOL_MODE_DONECLEANUP, + ADDTOOL_MODE_EXPORTRUN, + ADDTOOL_MODE_IMPORTRUN +} addtoolMode; + +pxConfig *addtoolConfig(pxConfig *config, int argc, char **argv); + +#endif // ADDTOOL_H Index: branches/eam_branches/20090820/ippTools/src/addtoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/addtoolConfig.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/addtoolConfig.c (revision 25766) @@ -0,0 +1,212 @@ +/* + * addtoolConfig.c + * + * Copyright (C) 2009 Joshua Hoblitt, Chris Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include + +#include "pxtools.h" +#include "pxadd.h" +#include "addtool.h" + +pxConfig *addtoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *definebyqueryArgs = psMetadataAlloc(); + pxcamSetSearchArgs(definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by chipRun reduction class", NULL); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -updaterun + // XXX need to allow multiple add_ids + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *updaterunArgs = psMetadataAlloc(); + pxcamSetSearchArgs(updaterunArgs); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by camRun label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by camRun state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + pxcamSetSearchArgs(pendingexpArgs); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // XXX is this used? psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class", 0, "search by class", NULL); + + // -addprocessedexp + psMetadata *addprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "define addtool ID (required)", 0); + + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_stars", 0, "define number of stars", 0); + + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-magicked", 0, "set magicked", 0); + // -processedexp + psMetadata *processedexpArgs = psMetadataAlloc(); + pxcamSetSearchArgs(processedexpArgs); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by addRun reduction class", NULL); + + psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-all", 0, "list everything without restriction", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); +/* psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); */ + + // -revertprocessedexp + // XXX need to allow multiple add_ids + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *revertprocessedexpArgs = psMetadataAlloc(); + pxcamSetSearchArgs(revertprocessedexpArgs); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by addRun reduction class", NULL); + + psMetadataAddBool(revertprocessedexpArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -updateprocessedexp + // XXX allow full search options? + psMetadata *updateprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by addtool ID", 0); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + + // -pendingcleanuprun + // XXX allow full search options? + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupexp + // XXX allow full search options? + psMetadata *pendingcleanupexpArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by addstar ID", 0); + psMetadataAddBool(pendingcleanupexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -exportrun + psMetadata *exportrunArgs = psMetadataAlloc(); + psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-add_id", 0, "export this addstar ID (required)", 0); + psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean", 0, "export tables as cleaned", false); + + // -importrun + psMetadata *importrunArgs = psMetadataAlloc(); + psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from cam stage", ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "change add run properties", ADDTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingexp", "show pending exps", ADDTOOL_MODE_PENDINGEXP, pendingexpArgs); + PXOPT_ADD_MODE("-addprocessedexp", "add a processed exp", ADDTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs); + PXOPT_ADD_MODE("-processedexp", "show processed exps", ADDTOOL_MODE_PROCESSEDEXP, processedexpArgs); + PXOPT_ADD_MODE("-revertprocessedexp", "change procesed exp properties", ADDTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpArgs); + PXOPT_ADD_MODE("-updateprocessedexp", "undo a processed exp", ADDTOOL_MODE_UPDATEPROCESSEDEXP,updateprocessedexpArgs); + PXOPT_ADD_MODE("-block", "set a label block", ADDTOOL_MODE_BLOCK, blockArgs); + PXOPT_ADD_MODE("-masked", "show blocked labels", ADDTOOL_MODE_MASKED, maskedArgs); + PXOPT_ADD_MODE("-unblock", "remove a label block", ADDTOOL_MODE_UNBLOCK, unblockArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", ADDTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); + PXOPT_ADD_MODE("-pendingcleanupexp", "show exps for cleanup runs", ADDTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpArgs); + PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", ADDTOOL_MODE_DONECLEANUP, donecleanupArgs); + PXOPT_ADD_MODE("-exportrun", "export run for import on other database", ADDTOOL_MODE_EXPORTRUN, exportrunArgs); + PXOPT_ADD_MODE("-importrun", "import run from metadata file", ADDTOOL_MODE_IMPORTRUN, importrunArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: branches/eam_branches/20090820/ippTools/src/camtool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/camtool.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/camtool.c (revision 25766) @@ -656,4 +656,21 @@ return false; } + +/* if (!pxaddQueueByCamID(config, */ +/* pendingRow->cam_id, */ +/* pendingRow->workdir, */ +/* pendingRow->label, */ +/* pendingRow->reduction, */ +/* pendingRow->dvodb */ +/* )) { */ +/* // rollback */ +/* if (!psDBRollback(config->dbh)) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* } */ +/* psError(PS_ERR_UNKNOWN, false, "failed to queue new addRun"); */ +/* psFree(pendingRow); */ +/* return false; */ +/* } */ + psFree(pendingRow); Index: branches/eam_branches/20090820/ippTools/src/chiptool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/chiptool.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/chiptool.c (revision 25766) @@ -54,4 +54,5 @@ static bool tofullimfileMode(pxConfig *config); static bool topurgedimfileMode(pxConfig *config); +static bool toscrubbedimfileMode(pxConfig *config); static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); @@ -93,4 +94,5 @@ MODECASE(CHIPTOOL_MODE_TOFULLIMFILE, tofullimfileMode); MODECASE(CHIPTOOL_MODE_TOPURGEDIMFILE, topurgedimfileMode); + MODECASE(CHIPTOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileMode); MODECASE(CHIPTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(CHIPTOOL_MODE_IMPORTRUN, importrunMode); @@ -601,5 +603,9 @@ PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); - PXOPT_COPY_S64(config->args, where, "-magicked", "chipRun.magicked", "=="); + PXOPT_COPY_S64(config->args, where, "-magicked", "chipProcessedImfile.magicked", "=="); + + PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); if (!psListLength(where->list) && @@ -629,4 +635,18 @@ // don't list faulted rows psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0"); + } + if (not_destreaked) { + if (destreaked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); + return false; + } + if (magicked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); + return false; + } + psStringAppend(&query, " AND chipProcessedImfile.magicked = 0"); + } + if (destreaked) { + psStringAppend(&query, " AND chipProcessedImfile.magicked != 0"); } @@ -1333,5 +1353,8 @@ return change_imfile_data_state(config, "purged", "goto_purged"); } - +static bool toscrubbedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "scrubbed", "goto_scrubbed"); +} bool exportrunMode(pxConfig *config) { Index: branches/eam_branches/20090820/ippTools/src/chiptool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/chiptool.h (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/chiptool.h (revision 25766) @@ -45,4 +45,5 @@ CHIPTOOL_MODE_TOFULLIMFILE, CHIPTOOL_MODE_TOPURGEDIMFILE, + CHIPTOOL_MODE_TOSCRUBBEDIMFILE, CHIPTOOL_MODE_EXPORTRUN, CHIPTOOL_MODE_IMPORTRUN Index: branches/eam_branches/20090820/ippTools/src/chiptoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/chiptoolConfig.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/chiptoolConfig.c (revision 25766) @@ -158,5 +158,8 @@ psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL); - psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-magicked", 0, "search by magicked status", false); + + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-destreaked", 0, "search for destreaked images", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for images that have not been destreaked", false); + psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-magicked", 0, "search by magicked value", 0); psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-all", 0, "list everything without search terms", false); @@ -247,4 +250,9 @@ psMetadataAddS64(topurgedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "chip ID to update", 0); psMetadataAddStr(topurgedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + // -toscrubbedimfile + psMetadata *toscrubbedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "chip ID to update", 0); + psMetadataAddStr(toscrubbedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); // -exportrun @@ -282,4 +290,5 @@ PXOPT_ADD_MODE("-tofullimfile", "set imfile state to full", CHIPTOOL_MODE_TOFULLIMFILE, tofullimfileArgs); PXOPT_ADD_MODE("-topurgedimfile", "set imfile state to purged", CHIPTOOL_MODE_TOPURGEDIMFILE, topurgedimfileArgs); + PXOPT_ADD_MODE("-toscrubbedimfile", "set imfile state to scrubbed", CHIPTOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", CHIPTOOL_MODE_EXPORTRUN, exportrunArgs); Index: branches/eam_branches/20090820/ippTools/src/dettool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/dettool.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/dettool.c (revision 25766) @@ -150,4 +150,5 @@ MODECASE(DETTOOL_MODE_UPDATEDETRUN, updatedetrunMode); MODECASE(DETTOOL_MODE_RERUN, rerunMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, pendingcleanup_detrunsummaryMode); // register MODECASE(DETTOOL_MODE_REGISTER_DETREND, register_detrendMode); @@ -1827,4 +1828,15 @@ if (!strcmp(data_state, "drop")) return true; if (!strcmp(data_state, "register")) return true; + // These are valid data states, and are necessary for the cleanup to work correctly. + if (!strcmp(data_state, "full")) return true; + if (!strcmp(data_state, "goto_cleaned")) return true; + if (!strcmp(data_state, "goto_scrubbed")) return true; + if (!strcmp(data_state, "goto_purged")) return true; + if (!strcmp(data_state, "cleaned")) return true; + if (!strcmp(data_state, "scrubbed")) return true; + if (!strcmp(data_state, "purged")) return true; + if (!strcmp(data_state, "error_cleaned")) return true; + if (!strcmp(data_state, "error_scrubbed")) return true; + if (!strcmp(data_state, "error_purged")) return true; psError(PS_ERR_UNKNOWN, true, "invalid data state: %s", data_state); @@ -1853,5 +1865,5 @@ if (!isValidDataState (data_state)) return false; - + char *query = "UPDATE detProcessedImfile SET data_state = '%s'" " WHERE det_id = %" PRId64 @@ -1900,5 +1912,5 @@ " WHERE det_id = %" PRId64 " AND iteration = %" PRId32 - " AND class_id = %s"; + " AND class_id = '%s'"; if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration, class_id)) { psError(PS_ERR_UNKNOWN, false, @@ -1922,6 +1934,7 @@ " WHERE det_id = %" PRId64 " AND iteration = %" PRId32 - " AND class_id = %s"; - if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration)) { + " AND class_id = '%s'"; +/* fprintf(stderr,"DETTOOL SAYS: %s\n",query); */ + if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration,class_id)) { psError(PS_ERR_UNKNOWN, false, "failed to change state for det_id %" PRId64 ", iteration %" PRId32, @@ -1944,5 +1957,5 @@ " WHERE det_id = %" PRId64 " AND iteration = %" PRId32 - " AND class_id = %s"; + " AND class_id = '%s'"; if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration, class_id)) { psError(PS_ERR_UNKNOWN, false, Index: branches/eam_branches/20090820/ippTools/src/dettool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/dettool.h (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/dettool.h (revision 25766) @@ -111,4 +111,5 @@ DETTOOL_MODE_REVERTDETRUNSUMMARY, DETTOOL_MODE_UPDATEDETRUNSUMMARY, + DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, DETTOOL_MODE_UPDATEDETRUN, DETTOOL_MODE_RERUN, @@ -209,4 +210,5 @@ bool revertdetrunsummaryMode(pxConfig *config); bool updatedetrunsummaryMode(pxConfig *config); +bool pendingcleanup_detrunsummaryMode(pxConfig *config); // other utilities Index: branches/eam_branches/20090820/ippTools/src/dettoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/dettoolConfig.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/dettoolConfig.c (revision 25766) @@ -41,11 +41,11 @@ // XXX EAM : is this used? does it make sense? psMetadata *pendingArgs = psMetadataAlloc(); - psMetadataAddS64(pendingArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exposure type", NULL); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst", 0, "search by camera", NULL); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope", 0, "search by telescope", NULL); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddS64(pendingArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exposure type", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst", 0, "search by camera", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope", 0, "search by telescope", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-uri", 0, "search by URL", NULL); - psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -definebytag @@ -125,4 +125,5 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_min", 0, "define min rotator position angle", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_min", 0, "define min solar angle", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_max", 0, "define max solar angle", NAN); @@ -137,4 +138,5 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_phase_max", 0, "define max moon phase", NAN); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "print the exposures that would be included in the detrend run and exit", false); @@ -207,7 +209,7 @@ // -addcorrectimfile psMetadata *addcorrectimfileArgs = psMetadataAlloc(); - psMetadataAddS64(addcorrectimfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); - psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID (required)", NULL); - psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-uri", 0, "define resid file URI", NULL); + psMetadataAddS64(addcorrectimfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID (required)", NULL); + psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-uri", 0, "define resid file URI", NULL); psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); @@ -306,5 +308,5 @@ psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_processedimfile @@ -366,5 +368,5 @@ psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); - psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_processedexp @@ -426,5 +428,5 @@ psMetadataAddS32(updatestackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); - psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_stacked @@ -478,5 +480,5 @@ psMetadataAddS32(updatenormalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_normalizedstat @@ -540,5 +542,5 @@ psMetadataAddS32(updatenormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_normalizedimfile @@ -598,5 +600,5 @@ psMetadataAddS64(updatenormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); psMetadataAddS32(updatenormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); - psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_normalizedexp @@ -678,5 +680,5 @@ psMetadataAddS64(updateresidimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_residimfile @@ -759,5 +761,5 @@ psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); psMetadataAddBool(updateresidexpArgs, PS_LIST_TAIL, "-reject", 0, "exposure is not to be stacked in the next iteration", false); - psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_residexp @@ -813,4 +815,13 @@ psMetadataAddBool(updatedetrunsummaryArgs, PS_LIST_TAIL, "-accept", 0, "declare that this detrun iteration is accepted as a master", false); psMetadataAddBool(updatedetrunsummaryArgs, PS_LIST_TAIL, "-reject", 0, "reject this detrun iteration as a master", false); + psMetadataAddStr(updatedetrunsummaryArgs, PS_LIST_TAIL, "-data_state", 0, "set the data state", NULL); + psMetadataAddS32(updatedetrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + + // -pendingcleanup_detrunsummary + psMetadata *pendingcleanup_detrunsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddU64(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -updatedetrun @@ -968,4 +979,6 @@ PXOPT_ADD_MODE("-updatedetrunsummary", "", DETTOOL_MODE_UPDATEDETRUNSUMMARY, updatedetrunsummaryArgs); + PXOPT_ADD_MODE("-pendingcleanup_detrunsummary", "", DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, pendingcleanup_detrunsummaryArgs); + PXOPT_ADD_MODE("-updatedetrun", "", DETTOOL_MODE_UPDATEDETRUN, updatedetrunArgs); PXOPT_ADD_MODE("-rerun", "", DETTOOL_MODE_RERUN, rerunArgs); Index: branches/eam_branches/20090820/ippTools/src/dettool_detrunsummary.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/dettool_detrunsummary.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/dettool_detrunsummary.c (revision 25766) @@ -336,5 +336,65 @@ } -// XXX need to add -data_state here +bool pendingcleanup_detrunsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_detrunsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return(false); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detRunSummary"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + // fprintf(stderr,">>>%s<<<\n",query); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return(true); + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pendingCleanupDetRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return(false); + } + psFree(output); + + return(true); +} + + +// preliminary code now. bool updatedetrunsummaryMode(pxConfig *config) { @@ -344,4 +404,7 @@ PXOPT_LOOKUP_BOOL(accept, config->args, "-accept", false); PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", ((accept == 0)&&(reject == 0)), false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", ((accept == 0)&&(reject == 0)), false); + if (accept && reject) { @@ -350,13 +413,109 @@ } - if (!(accept || reject)) { - psError(PS_ERR_UNKNOWN, true, "either -accept or -reject is required"); - return false; - } - - char *query = "UPDATE detRunSummary SET accept = %d WHERE det_id = %"PRId64; - if (!p_psDBRunQueryF(config->dbh, query, accept, det_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; + if (!(accept || reject || (data_state != NULL))) { + psError(PS_ERR_UNKNOWN, true, "either -accept or -reject is required if -data_state is not supplied"); + return false; + } + + if (accept || reject) { + char *query = "UPDATE detRunSummary SET accept = %d WHERE det_id = %"PRId64; + if (!p_psDBRunQueryF(config->dbh, query, accept, det_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + else { + PS_ASSERT_PTR_NON_NULL(data_state,false); + +/* if (!isValidDataState(data_state)) return false; */ + + + + char *query_detRunSummary = "UPDATE detRunSummary SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detRunSummary, data_state, det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return false; + } + + /* This check allows the one update to flag everything for cleanup. The check for full is only temporary while I test for bugs. */ + if ((!strncmp(data_state,"goto_",5) + // || (!strcmp(data_state,"full")) + // || (!strcmp(data_state,"cleaned")) + )) { + char *query_detProcessedImfile = "UPDATE detProcessedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query_detProcessedImfile,data_state,det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detProcessedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detProcessedExp = "UPDATE detProcessedExp SET data_state = '%s'" + " WHERE det_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query_detProcessedExp,data_state,det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detProcessedExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detNormalizedImfile = "UPDATE detNormalizedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detNormalizedImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detNormalizedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detNormalizedStatImfile = "UPDATE detNormalizedStatImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detNormalizedStatImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detNormalizedStatImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detNormalizedExp = "UPDATE detNormalizedExp SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detNormalizedExp,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detNormalizedExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detResidImfile = "UPDATE detResidImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detResidImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detResidImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detResidExp = "UPDATE detResidExp SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detResidExp,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detResidExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detStackedImfile = "UPDATE detStackedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detStackedImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detStackedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + } + /* End if */ + } Index: branches/eam_branches/20090820/ippTools/src/dettool_processedimfile.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/dettool_processedimfile.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/dettool_processedimfile.c (revision 25766) @@ -314,5 +314,5 @@ PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - +/* int i; */ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); @@ -345,4 +345,5 @@ return false; } +/* fprintf(stderr,"DETTOOL:procimfile: %s\n",query); */ psFree(query); @@ -350,8 +351,10 @@ if (!output) { psError(PS_ERR_UNKNOWN, false, "database error"); +/* fprintf(stderr,"WTF !output?\n"); */ return false; } if (!psArrayLength(output)) { psTrace("dettool", PS_LOG_INFO, "no rows found"); +/* fprintf(stderr,"WTF no rows??\n"); */ psFree(output); return true; @@ -359,4 +362,6 @@ // negative simple so the default is true +/* i = (int) ippdbPrintMetadatas(stdout, output, "detCleanupProcessedImfile", !simple); */ +/* fprintf(stderr,">>%d<<\n",i); */ if (!ippdbPrintMetadatas(stdout, output, "detCleanupProcessedImfile", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); @@ -364,4 +369,6 @@ return false; } +/* fprintf(stderr,"DETTOOL:procimfile: %s\n",output); */ +/* psFree(output); */ psFree(output); Index: branches/eam_branches/20090820/ippTools/src/difftool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/difftool.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/difftool.c (revision 25766) @@ -37,4 +37,5 @@ static bool todiffskyfileMode(pxConfig *config); static bool adddiffskyfileMode(pxConfig *config); +static bool advanceMode(pxConfig *config); static bool diffskyfileMode(pxConfig *config); static bool revertdiffskyfileMode(pxConfig *config); @@ -50,5 +51,4 @@ static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, psS64 magicked); -static bool diffRunComplete(pxConfig *config); # define MODECASE(caseName, func) \ @@ -76,8 +76,9 @@ MODECASE(DIFFTOOL_MODE_TODIFFSKYFILE, todiffskyfileMode); MODECASE(DIFFTOOL_MODE_ADDDIFFSKYFILE, adddiffskyfileMode); + MODECASE(DIFFTOOL_MODE_ADVANCE, advanceMode); MODECASE(DIFFTOOL_MODE_DIFFSKYFILE, diffskyfileMode); MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileMode); MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunMode); - MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackMode); + MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackMode); MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP, definewarpwarpMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); @@ -555,12 +556,4 @@ } - if (!diffRunComplete(config)) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - // point of no return if (!psDBCommit(config->dbh)) { @@ -569,4 +562,95 @@ } + + return true; +} + +static bool advanceMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + // look for completed diffRuns + psString query = pxDataGet("difftool_completed_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psString whereString = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereString, "\n AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereString); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, query, whereString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(whereString); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + psFree(whereString); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); + psS64 magicked = psMetadataLookupS64(NULL, row, "magicked"); + + // set diffRun.state to 'full' + if (!setdiffRunState(config, diff_id, "full", magicked)) { + psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, + diff_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } return true; @@ -581,13 +665,34 @@ PXOPT_COPY_S64(config->args, where, "-diff_id", "diffSkyfile.diff_id", "=="); PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "=="); - PXOPT_COPY_S64(config->args, where, "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "=="); PXOPT_COPY_STR(config->args, where, "-tess_id", "diffRun.tess_id", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "diffSkyfile.fault", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "rawInput.exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "rawInput.exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-warp_id", "warpInput.warp_id", "=="); - + PXOPT_COPY_S64(config->args, where, "-magicked", "diffSkyfile.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "LIKE"); + + PXOPT_LOOKUP_BOOL(template, config->args, "-template", false); + if (!template) { + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawInput.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawInput.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_id", "warpInput.warp_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawInput.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawInput.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawInput.filter", "=="); + } else { + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawTemplate.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawTemplate.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_id", "warpTemplate.warp_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawTemplate.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawTemplate.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawTemplate.filter", "=="); + } + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); psString query = pxDataGet("difftool_skyfile.sql"); @@ -601,6 +706,24 @@ psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); + } else if (!all) { + psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); } psFree(where); + + if (not_destreaked) { + if (destreaked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); + return false; + } + if (magicked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); + return false; + } + psStringAppend(&query, " AND diffSkyfile.magicked = 0"); + } + if (destreaked) { + psStringAppend(&query, " AND diffSkyfile.magicked != 0"); + } + // treat limit == 0 as "no limit" @@ -1810,49 +1933,4 @@ } -static bool diffRunComplete(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // look for completed diffRuns - psString query = pxDataGet("difftool_completed_runs.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("difftool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *row = output->data[i]; - - psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); - psS64 magicked = psMetadataLookupS64(NULL, row, "magicked"); - - // set diffRun.state to 'stop' - if (!setdiffRunState(config, diff_id, "full", magicked)) { - psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, - diff_id); - psFree(output); - return false; - } - } - - return true; -} bool exportrunMode(pxConfig *config) Index: branches/eam_branches/20090820/ippTools/src/difftool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/difftool.h (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/difftool.h (revision 25766) @@ -31,4 +31,5 @@ DIFFTOOL_MODE_TODIFFSKYFILE, DIFFTOOL_MODE_ADDDIFFSKYFILE, + DIFFTOOL_MODE_ADVANCE, DIFFTOOL_MODE_DIFFSKYFILE, DIFFTOOL_MODE_REVERTDIFFSKYFILE, Index: branches/eam_branches/20090820/ippTools/src/difftoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/difftoolConfig.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/difftoolConfig.c (revision 25766) @@ -115,16 +115,32 @@ psMetadataAddBool(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked state", false); + // -advance + psMetadata *advanceArgs = psMetadataAlloc(); + psMetadataAddS64(advanceArgs, PS_LIST_TAIL, "-diff_id", 0, "select by diff ID", 0); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "select by label", NULL); + psMetadataAddS32(advanceArgs, PS_LIST_TAIL, "-limit", 0, "limit number of results", 0); + // -diffskyfile psMetadata *diffskyfileArgs = psMetadataAlloc(); - psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); - psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID", NULL); - psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-diff_skyfile_id", 0, "search by diff_skyfile_id ID", 0); - psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID", NULL); - psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0); - psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-exp_name", 0, "define exposure name", NULL); - psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-warp_id", 0, "define warp_id", NULL); - psMetadataAddU64(diffskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddS16(diffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-diff_skyfile_id", 0, "search by diff_skyfile_id ID", 0); + psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", NULL); + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-template", 0, "apply exposure args to template of bothways diff", false); + psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", NULL); + psMetadataAddTime(diffskyfileArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(diffskyfileArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); + psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "search by magicked value", 0); + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-destreaked", 0, "search for destreaked images", false); + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for images that are not destreaked", false); + psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by diffRun label (LIKE comparison)", NULL); + psMetadataAddS16(diffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(diffskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -revertdiffskyfile @@ -267,4 +283,5 @@ PXOPT_ADD_MODE("-todiffskyfile", "", DIFFTOOL_MODE_TODIFFSKYFILE, todiffskyfileArgs); PXOPT_ADD_MODE("-adddiffskyfile", "", DIFFTOOL_MODE_ADDDIFFSKYFILE, adddiffskyfileArgs); + PXOPT_ADD_MODE("-advance", "", DIFFTOOL_MODE_ADVANCE, advanceArgs); PXOPT_ADD_MODE("-diffskyfile", "", DIFFTOOL_MODE_DIFFSKYFILE, diffskyfileArgs); PXOPT_ADD_MODE("-revertdiffskyfile","", DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileArgs); Index: branches/eam_branches/20090820/ippTools/src/disttool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/disttool.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/disttool.c (revision 25766) @@ -2,5 +2,5 @@ * disttool.c * - * Copyright (C) 2008 + * Copyright (C) 2008-2009 * * This program is free software; you can redistribute it and/or modify it @@ -37,4 +37,5 @@ static bool pendingcomponentMode(pxConfig *config); static bool addprocessedcomponentMode(pxConfig *config); +static bool revertcomponentMode(pxConfig *config); static bool processedcomponentMode(pxConfig *config); static bool toadvanceMode(pxConfig *config); @@ -51,7 +52,4 @@ static bool listtargetMode(pxConfig *config); -static bool definedsproductMode(pxConfig *config); -static bool updatedsproductMode(pxConfig *config); - static bool definedestinationMode(pxConfig *config); static bool updatedestinationMode(pxConfig *config); @@ -59,4 +57,5 @@ static bool defineinterestMode(pxConfig *config); static bool updateinterestMode(pxConfig *config); +static bool listinterestsMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -86,4 +85,5 @@ MODECASE(DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentMode); MODECASE(DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentMode); + MODECASE(DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentMode); MODECASE(DISTTOOL_MODE_TOADVANCE, toadvanceMode); MODECASE(DISTTOOL_MODE_PENDINGFILESET, pendingfilesetMode); @@ -97,10 +97,9 @@ MODECASE(DISTTOOL_MODE_UPDATETARGET, updatetargetMode); MODECASE(DISTTOOL_MODE_LISTTARGET, listtargetMode); - MODECASE(DISTTOOL_MODE_DEFINEDSPRODUCT, definedsproductMode); - MODECASE(DISTTOOL_MODE_UPDATEDSPRODUCT, updatedsproductMode); MODECASE(DISTTOOL_MODE_DEFINEDESTINATION, definedestinationMode); MODECASE(DISTTOOL_MODE_UPDATEDESTINATION, updatedestinationMode); MODECASE(DISTTOOL_MODE_DEFINEINTEREST, defineinterestMode); MODECASE(DISTTOOL_MODE_UPDATEINTEREST, updateinterestMode); + MODECASE(DISTTOOL_MODE_LISTINTERESTS, listinterestsMode); default: psAbort("invalid option (this should not happen)"); @@ -151,4 +150,5 @@ stage, stage_id, + 0, set_label, outroot, @@ -223,5 +223,6 @@ } } else if (!strcmp(stage, "camera")) { - magicRunType = "chipRun"; // This is used below to set the magicked business + magicRunType = "camRun"; // This is used below to set the magicked business + runJoinStr = "camRun.cam_id"; query = pxDataGet("disttool_definebyquery_camera.sql"); if (!query) { @@ -288,5 +289,5 @@ psStringAppend(&query, " AND (stackRun.label = '%s')", label); } - // stack stage doesn't require magic (perhaps let the script do this? + // stack stage doesn't require magic no_magic = true; } else { @@ -306,24 +307,15 @@ if (!no_magic) { - psStringAppend(&query, " AND (distTarget.clean OR %s.magicked)", magicRunType); + psStringAppend(&query, " AND (%s.magicked)", magicRunType); // is selecting by magic_ds_id really interesting? if (magic_ds_id) { - if (strcmp(stage, "camera")) { - // stage other than camera - if (!runJoinStr) { - psError(PS_ERR_PROGRAMMING, true, "cannot select by magic_ds_id for stage: %s", stage); - psFree(query); - return false; - } - psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = distTarget.stage" + if (!runJoinStr) { + psError(PS_ERR_PROGRAMMING, true, "cannot select by magic_ds_id for stage: %s", stage); + psFree(query); + return false; + } + psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = distTarget.stage" " AND magicDSRun.stage_id = %s", runJoinStr); - } else { - // camera masks are magicked when the chipRun is magicked - // XXX: This is confusing. Is it dangerous? - // Maybe I should add a magicked bit to camRun. Note this isn't - psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = 'chip'" - " AND magicDSRun.stage_id = chipRun.chip_id"); - } psStringAppend(&query, " AND (magicDSRun.state = 'full' AND magicDSRun.re_place AND (magic_ds_id = %" PRId64 "))", magic_ds_id); } @@ -377,4 +369,5 @@ psString run_tag = psMetadataLookupStr(NULL, md, "run_tag"); psS64 stage_id = psMetadataLookupS64(NULL, md, "stage_id"); + psS64 magic_ds_id = psMetadataLookupS64(NULL, md, "magicked"); psS64 target_id = psMetadataLookupS64(NULL, md, "target_id"); psString target_label = psMetadataLookupStr(NULL, md, "label"); @@ -395,4 +388,5 @@ stage, stage_id, + magic_ds_id, new_label, outroot, @@ -500,8 +494,5 @@ PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - // we need to disambiguate fault so make a copy of the where list before adding fault - psMetadata *whereComponent = psMetadataCopy(NULL, where); PXOPT_COPY_S16(config->args, where, "-fault", "distRun.fault", "=="); - PXOPT_COPY_S16(config->args, whereComponent, "-fault", "distComponent.fault", "=="); // It might be useful to be able to query by the parameters of the underlying runs @@ -513,84 +504,77 @@ } - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psString query = pxDataGet("disttool_revertrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + int numUpdated = psDBAffectedRows(config->dbh); + + psLogMsg("disttool", PS_LOG_INFO, "Updated %d dist runs", numUpdated); + + return true; +} + +static bool revertcomponentMode(pxConfig *config) +{ + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dist_id", "distRun.dist_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");; + PXOPT_COPY_STR(config->args, where, "-component", "component", "==");; + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + PXOPT_COPY_S16(config->args, where, "-fault", "distComponent.fault", "=="); + + // It might be useful to be able to query by the parameters of the underlying runs + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - return false; - } - - // Update state to 'new' - int numUpdated; // Number updated - { - psString query = pxDataGet("disttool_revertrun_update.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - numUpdated = psDBAffectedRows(config->dbh); - } - - psLogMsg("disttool", PS_LOG_INFO, "Updated %d dist runs", numUpdated); - - // Delete product - int numDeleted; // Number deleted - { - psString query = pxDataGet("disttool_revertrun_delete.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(whereComponent->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(whereComponent, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - numDeleted = psDBAffectedRows(config->dbh); - } + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("disttool_revertcomponent.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + int numDeleted = psDBAffectedRows(config->dbh); psLogMsg("disttool", PS_LOG_INFO, "Deleted %d distComponents", numDeleted); - psFree(where); - psFree(whereComponent); - - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - return true; } @@ -600,16 +584,18 @@ PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); - PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "distRun.label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // look for "inputs" that need to processed - psString query = pxDataGet("disttool_pendingcomponent.sql"); + psString queryFile = NULL; + psStringAppend(&queryFile, "disttool_pending_%s.sql", stage); + psString query = pxDataGet(queryFile); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", queryFile); return false; } @@ -617,5 +603,5 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } @@ -629,28 +615,5 @@ } - // the query has where hooks for each stage. - // right now we aren't using them. - // XXX: I think that I want to change the query from a union of selects on the various - // stages to separate queries. As it is pending data at the later stages of the pipline - // will get blocked by pending earlier stages - psString raw_where = ""; - psString raw_clean_where = ""; - psString chip_where = ""; - psString camera_where = ""; - psString fake_where = ""; - psString warp_where = ""; - psString diff_where = ""; - psString stack_where = ""; - - if (!p_psDBRunQueryF(config->dbh, - query, - raw_where, - raw_clean_where, - chip_where, - camera_where, - fake_where, - warp_where, - diff_where, - stack_where)) { + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -868,4 +831,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "distRun.stage", "=="); pxAddLabelSearchArgs (config, where, "-label", "distRun.label", "=="); @@ -939,5 +903,5 @@ // required values PXOPT_LOOKUP_S64(dist_id, config->args, "-dist_id", true, false); - PXOPT_LOOKUP_S64(prod_id, config->args, "-prod_id", true, false); + PXOPT_LOOKUP_S64(dest_id, config->args, "-dest_id", true, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); @@ -949,5 +913,5 @@ 0, // fs_id dist_id, - prod_id, + dest_id, name, "full", @@ -964,5 +928,5 @@ PXOPT_COPY_S64(config->args, where, "-fs_id", "fs_id", "=="); PXOPT_COPY_S64(config->args, where, "-dist_id", "rcDSFileset.dist_id", "=="); - PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");; PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); @@ -1092,5 +1056,4 @@ PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); - PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "=="); PXOPT_COPY_S64(config->args, where, "-target_id","target_id", "=="); PXOPT_COPY_S64(config->args, where, "-fs_id", "fs_id", "=="); @@ -1433,99 +1396,12 @@ } -static bool definedsproductMode(pxConfig *config) +static bool definedestinationMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); // required - PXOPT_LOOKUP_STR(name, config->args, "-name", true, false); - PXOPT_LOOKUP_STR(dbname, config->args, "-ds_dbname", true, false); - PXOPT_LOOKUP_STR(dbhost, config->args, "-ds_dbhost", true, false); - - // XXX: should we insure that these names do not contatin any whitespace? - - rcDSProductRow *row = rcDSProductRowAlloc( - 0, // prod_id - name, - dbname, - dbhost - ); - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to allocate rcDSProduct object"); - return false; - } - if (!rcDSProductInsertObject(config->dbh, row)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(row); - return false; - } - - // get the assigned target_id - row->prod_id = psDBLastInsertID(config->dbh); - - if (!rcDSProductPrintObject(stdout, row, true)) { - psError(PS_ERR_UNKNOWN, false, "failed to print object"); - psFree(row); - return false; - } - - psFree(row); - - return true; -} -static bool updatedsproductMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "=="); - - PXOPT_LOOKUP_STR(dbname, config->args, "-ds_dbname", false, false); - PXOPT_LOOKUP_STR(dbhost, config->args, "-ds_dbhost", false, false); - - if (!(dbname || dbhost)) { - psError(PS_ERR_UNKNOWN, true, "one or more of dbname or dbhost is required"); - psFree(where); - return false; - } - psString query = psStringCopy("UPDATE rcDSProduct SET"); - psString sep = ""; - if (dbname) { - psStringAppend(&query, " dbname = '%s'", dbname); - sep = ","; - } - if (dbhost) { - psStringAppend(&query, " %s dbhost = '%s'", sep, dbhost); - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); - psFree(whereClause); - } else { - psError(PS_ERR_UNKNOWN, true, "search parameters are required"); - psFree(where); - psFree(query); - return false; - } - psFree(where); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - return true; -} - -static bool definedestinationMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // required - PXOPT_LOOKUP_S64(prod_id, config->args, "-prod_id", true, false); PXOPT_LOOKUP_STR(name, config->args, "-name", true, false); + PXOPT_LOOKUP_STR(dbname, config->args, "-ds_dbname", true, false); + PXOPT_LOOKUP_STR(dbhost, config->args, "-ds_dbhost", true, false); // optional @@ -1539,9 +1415,10 @@ rcDestinationRow *row = rcDestinationRowAlloc( 0, // dest_id - prod_id, name, status_uri, comment, last_fileset, + dbname, + dbhost, state ? state : "enabled" ); @@ -1577,5 +1454,4 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); - PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "=="); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); @@ -1599,5 +1475,5 @@ // last_fileset normally gets set by updatercrunMode // Allowing it to be set here might cause problems - // especially since we are allowing selection by prod_id + // especially since we are allowing selection by dest_id if (last_fileset) { psStringAppend(&query, " %s last_fileset = '%s'", sep, last_fileset); @@ -1631,40 +1507,102 @@ PS_ASSERT_PTR_NON_NULL(config, false); - // required - PXOPT_LOOKUP_S64(dest_id, config->args, "-dest_id", true, false); - PXOPT_LOOKUP_S64(target_id, config->args, "-target_id", true, false); + // one of these is required + PXOPT_LOOKUP_S64(dest_id, config->args, "-dest_id", false, false); + PXOPT_LOOKUP_STR(dest_name, config->args, "-dest_name", false, false); + if (!dest_id && !dest_name) { + psError(PS_ERR_UNKNOWN, true, "either dest_id or dest_name is required"); + return false; + } + + // either target_id or stage and label are required + PXOPT_LOOKUP_S64(target_id, config->args, "-target_id", false, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-filter", false, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + if (!target_id) { + bool error = false; + if (!stage) { + psError(PS_ERR_UNKNOWN, true, "stage is required if target_id is not supplied"); + error = true; + } + if (!label) { + psError(PS_ERR_UNKNOWN, !error, "label is required if target_id is not supplied"); + error = true; + } + if (error) { + return false; + } + } // optional + PXOPT_LOOKUP_S64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - - // XXX: should we insure that these names do not contatin any whitespace? - - rcInterestRow *row = rcInterestRowAlloc( - 0, // int_id - dest_id, - target_id, - state ? state : "enabled" - ); - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to allocate rcInterest object"); - return false; - } - if (!rcInterestInsertObject(config->dbh, row)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(row); - return false; - } - - // get the assigned target_id - row->int_id = psDBLastInsertID(config->dbh); - - if (!rcInterestPrintObject(stdout, row, true)) { - psError(PS_ERR_UNKNOWN, false, "failed to print object"); - psFree(row); - return false; - } - - psFree(row); + if (state) { + if (strcmp(state, "enabled") && strcmp(state, "disabled")) { + psError(PS_ERR_PROGRAMMING, true, "state must be enabled or disabled"); + return false; + } + } else { + // default state + state = "enabled"; + } + + // now that we've done all of our argument checking, copy the values to where + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); + PXOPT_COPY_STR(config->args, where, "-dest_name", "rcDestination.name", "=="); + PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + + psString query = pxDataGet("disttool_defineinterest.sql"); + + if (!psListLength(where->list)) { + // can't get here + psError(PS_ERR_PROGRAMMING, true, "search parameters are required"); + psFree(where); + psFree(query); + return false; + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + psFree(where); + if (clean) { + psStringAppend(&query, " AND (distTarget.clean)"); + } else { + psStringAppend(&query, " AND (!distTarget.clean)"); + } + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + { + // psStringSubstitute fails unless the input is a psString which it determines by + // comparing the memory blocks free function to an expected value. + // pxDataGet uses psSlurp which leaves a different free function on the memory block. + // To work around this make a copy of the query before doing the substitution. + psString queryCopy = psStringCopy(query); + psFree(query); + query = queryCopy; + } + // change the @STATE@ in the sql file to our state + if (!psStringSubstitute(&query, state, "@STATE@")) { + psError(PS_ERR_UNKNOWN, false, "failed to substitute state string"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + int numInserted = psDBAffectedRows(config->dbh); + printf("inserted %d rows into rcInterest\n", numInserted); return true; @@ -1710,3 +1648,83 @@ return true; } - +static bool listinterestsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-int_id", "int_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); + PXOPT_COPY_STR(config->args, where, "-dest_name", "name", "=="); + PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + PXOPT_LOOKUP_BOOL(full, config->args, "-full", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (clean && full) { + psError(PS_ERR_UNKNOWN, false, "can't select both -clean and -full"); + return false; + } + + psString query = pxDataGet("disttool_listinterests.sql"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + if (clean) { + psStringAppend(&query, " AND (clean)"); + } else if (full) { + psStringAppend(&query, " AND (!clean)"); + } + } else if (clean) { + psStringAppend(&query, " WHERE clean"); + } else if (full) { + psStringAppend(&query, " WHERE !clean"); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("disttool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "rcInterest", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} Index: branches/eam_branches/20090820/ippTools/src/disttool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/disttool.h (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/disttool.h (revision 25766) @@ -31,4 +31,5 @@ DISTTOOL_MODE_PENDINGCOMPONENT, DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, + DISTTOOL_MODE_REVERTCOMPONENT, DISTTOOL_MODE_PROCESSEDCOMPONENT, DISTTOOL_MODE_TOADVANCE, @@ -42,6 +43,4 @@ DISTTOOL_MODE_DEFINEDESTINATION, DISTTOOL_MODE_UPDATEDESTINATION, - DISTTOOL_MODE_DEFINEDSPRODUCT, - DISTTOOL_MODE_UPDATEDSPRODUCT, DISTTOOL_MODE_DEFINETARGET, DISTTOOL_MODE_UPDATETARGET, @@ -49,4 +48,5 @@ DISTTOOL_MODE_DEFINEINTEREST, DISTTOOL_MODE_UPDATEINTEREST, + DISTTOOL_MODE_LISTINTERESTS, } disttoolMode; Index: branches/eam_branches/20090820/ippTools/src/disttoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/disttoolConfig.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/disttoolConfig.c (revision 25766) @@ -95,6 +95,6 @@ // -pendingcomponent psMetadata *pendingcomponentArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-stage", 0, "limit results to runs for stage (required)", NULL); psMetadataAddS64(pendingcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); - psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-stage", 0, "limit results to runs for stage", NULL); psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit results to label", NULL); psMetadataAddU64(pendingcomponentArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -111,4 +111,15 @@ psMetadataAddS32(addprocessedcomponentArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + // -revertcomponent + psMetadata *revertcomponentArgs = psMetadataAlloc(); + psMetadataAddS64(revertcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); + psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL); + psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); + psMetadataAddS64(revertcomponentArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id", 0); + psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-state", 0, "define state", NULL); + psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddS16(revertcomponentArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + psMetadataAddBool(revertcomponentArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted runs", NULL); + // -processedcomponent psMetadata *processedcomponentArgs = psMetadataAlloc(); @@ -135,5 +146,5 @@ psMetadata *addfilesetArgs = psMetadataAlloc(); psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); - psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id", 0); + psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0); psMetadataAddStr(addfilesetArgs, PS_LIST_TAIL, "-name", 0, "define file name", NULL); psMetadataAddS32(addfilesetArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); @@ -150,5 +161,5 @@ psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-fs_id", 0, "define fs_id", 0); psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); - psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-prod_id", 0, "define dist_id", 0); + psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-dest_id", 0, "define dist_id", 0); psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-stage_id",0, "define stage_id", 0); @@ -162,5 +173,4 @@ psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0); - psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id", 0); psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-fs_id", 0, "define fs_id", 0); @@ -189,22 +199,9 @@ psMetadataAddBool(revertrcrunArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted runs", NULL); - // -definedsproduct - psMetadata *definedsproductArgs = psMetadataAlloc(); - psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-name", 0, "define product name", NULL); - psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name", NULL); - psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host", NULL); - - // -updatedsproduct - // does this mode make sense? - psMetadata *updatedsproductArgs = psMetadataAlloc(); - psMetadataAddS64(updatedsproductArgs, PS_LIST_TAIL, "-prod_id", 0, "select by prod_id", 0); - // can't select by name because it isn't necssarily unique - psMetadataAddStr(updatedsproductArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name", NULL); - psMetadataAddStr(updatedsproductArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host", NULL); - // -definedestination psMetadata *definedestinationArgs = psMetadataAlloc(); - psMetadataAddS64(definedestinationArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id (required)", 0); psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-name", 0, "define destination name (required)", NULL); + psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name (required)", NULL); + psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host (required)", NULL); psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-status_uri", 0, "define status_uri", NULL); psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); @@ -215,10 +212,7 @@ psMetadata *updatedestinationArgs = psMetadataAlloc(); psMetadataAddS64(updatedestinationArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0); - psMetadataAddS64(updatedestinationArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id", 0); psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-name", 0, "define destination name", NULL); psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-status_uri", 0, "define status_uri", NULL); psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); -// last_fileset gets updated by -updatercrun -// psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-set_last_fileset",0, "define last_fileset", NULL); psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); @@ -254,7 +248,13 @@ // -defineinterest psMetadata *defineinterestArgs = psMetadataAlloc(); - psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id (required)", 0); - psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id (required)", 0); + psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-dest_name", 0, "define destination name (LIKE comparison)", NULL); + psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-filter", 0, "define filter (LIKE comparison)", NULL); + psMetadataAddBool(defineinterestArgs, PS_LIST_TAIL,"-clean", 0, "list clean targets", false); psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + psMetadataAddU64(defineinterestArgs, PS_LIST_TAIL, "-limit", 0, "limit number of targets listed to N", 0); // -updateinterest @@ -265,4 +265,19 @@ psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-set_state", 0, "define state (required)", NULL); + // -listinterests + psMetadata *listinterestsArgs = psMetadataAlloc(); + psMetadataAddS64(listinterestsArgs, PS_LIST_TAIL, "-target_id", 0, "list interests with target_id", 0); + psMetadataAddS64(listinterestsArgs, PS_LIST_TAIL, "-dest_id", 0, "list interests with dest_id", 0); + psMetadataAddS64(listinterestsArgs, PS_LIST_TAIL, "-int_id", 0, "list interests with int_id", 0); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-dest_name", 0, "list interests for destinationn name)", NULL); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-label", 0, "list interests for label (LIKE comparison)", NULL); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-filter", 0, "list interests by filter (LIKE comparison)", NULL); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-stage", 0, "list interests for stage", NULL); + psMetadataAddBool(listinterestsArgs, PS_LIST_TAIL,"-clean", 0, "list clean interests", false); + psMetadataAddBool(listinterestsArgs, PS_LIST_TAIL,"-full", 0, "list full interests", false); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-state", 0, "list interests in state", NULL); + psMetadataAddU64(listinterestsArgs, PS_LIST_TAIL, "-limit", 0, "limit number of interests listed to N", 0); + psMetadataAddBool(listinterestsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadata *argSets = psMetadataAlloc(); psMetadata *modes = psMetadataAlloc(); @@ -272,6 +287,7 @@ PXOPT_ADD_MODE("-updaterun", "", DISTTOOL_MODE_UPDATERUN, updaterunArgs); PXOPT_ADD_MODE("-revertrun", "", DISTTOOL_MODE_REVERTRUN, revertrunArgs); - PXOPT_ADD_MODE("-pendingcomponent", "", DISTTOOL_MODE_PENDINGCOMPONENT, pendingcomponentArgs); - PXOPT_ADD_MODE("-addprocessedcomponent", "", DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentArgs); + PXOPT_ADD_MODE("-pendingcomponent", "", DISTTOOL_MODE_PENDINGCOMPONENT, pendingcomponentArgs); + PXOPT_ADD_MODE("-addprocessedcomponent", "", DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentArgs); + PXOPT_ADD_MODE("-revertcomponent", "", DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentArgs); PXOPT_ADD_MODE("-processedcomponent", "", DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentArgs); PXOPT_ADD_MODE("-toadvance", "", DISTTOOL_MODE_TOADVANCE, toadvanceArgs); @@ -284,8 +300,4 @@ PXOPT_ADD_MODE("-pendingdest", "", DISTTOOL_MODE_PENDINGDEST, pendingdestArgs); - PXOPT_ADD_MODE("-definedsproduct", "", DISTTOOL_MODE_DEFINEDSPRODUCT, definedsproductArgs); - PXOPT_ADD_MODE("-updatedsproduct", "", DISTTOOL_MODE_UPDATEDSPRODUCT, updatedsproductArgs); -// PXOPT_ADD_MODE("-listdsproduct", "", DISTTOOL_MODE_LISTDSPRODUCT, updatedsproductArgs); - PXOPT_ADD_MODE("-definedestination", "", DISTTOOL_MODE_DEFINEDESTINATION, definedestinationArgs); PXOPT_ADD_MODE("-updatedestination", "", DISTTOOL_MODE_UPDATEDESTINATION, updatedestinationArgs); @@ -298,5 +310,5 @@ PXOPT_ADD_MODE("-defineinterest", "", DISTTOOL_MODE_DEFINEINTEREST, defineinterestArgs); PXOPT_ADD_MODE("-updateinterest", "", DISTTOOL_MODE_UPDATEINTEREST, updateinterestArgs); -// PXOPT_ADD_MODE("-listinterest", "", DISTTOOL_MODE_LISTINTEREST, listinterestArgs); + PXOPT_ADD_MODE("-listinterests", "", DISTTOOL_MODE_LISTINTERESTS, listinterestsArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/eam_branches/20090820/ippTools/src/faketool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/faketool.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/faketool.c (revision 25766) @@ -52,4 +52,5 @@ static bool tofullimfileMode(pxConfig *config); static bool topurgedimfileMode(pxConfig *config); +static bool toscrubbedimfileMode(pxConfig *config); static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); @@ -91,4 +92,5 @@ MODECASE(FAKETOOL_MODE_TOFULLIMFILE, tofullimfileMode); MODECASE(FAKETOOL_MODE_TOPURGEDIMFILE, topurgedimfileMode); + MODECASE(FAKETOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileMode); MODECASE(FAKETOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(FAKETOOL_MODE_IMPORTRUN, importrunMode); @@ -1251,4 +1253,8 @@ return change_imfile_data_state(config, "purged", "goto_purged"); } +static bool toscrubbedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "scrubbed", "goto_scrubbed"); +} bool exportrunMode(pxConfig *config) Index: branches/eam_branches/20090820/ippTools/src/faketool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/faketool.h (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/faketool.h (revision 25766) @@ -45,4 +45,5 @@ FAKETOOL_MODE_TOFULLIMFILE, FAKETOOL_MODE_TOPURGEDIMFILE, + FAKETOOL_MODE_TOSCRUBBEDIMFILE, FAKETOOL_MODE_EXPORTRUN, FAKETOOL_MODE_IMPORTRUN Index: branches/eam_branches/20090820/ippTools/src/faketoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/faketoolConfig.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/faketoolConfig.c (revision 25766) @@ -301,4 +301,9 @@ psMetadataAddStr(topurgedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + // -toscrubbedimfile + psMetadata *toscrubbedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "fake ID to update", 0); + psMetadataAddStr(toscrubbedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); @@ -337,4 +342,5 @@ PXOPT_ADD_MODE("-tofullimfile", "set imfile state to full", FAKETOOL_MODE_TOFULLIMFILE, tofullimfileArgs); PXOPT_ADD_MODE("-topurgedimfile", "set imfile state to purged", FAKETOOL_MODE_TOPURGEDIMFILE, topurgedimfileArgs); + PXOPT_ADD_MODE("-toscrubbedimfile", "set imfile state to scrubbed", FAKETOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", FAKETOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", FAKETOOL_MODE_IMPORTRUN, importrunArgs); Index: branches/eam_branches/20090820/ippTools/src/magicdstool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/magicdstool.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/magicdstool.c (revision 25766) @@ -37,12 +37,12 @@ static bool todestreakMode(pxConfig *config); static bool adddestreakedfileMode(pxConfig *config); +static bool advancerunMode(pxConfig *config); static bool revertdestreakedfileMode(pxConfig *config); static bool getskycellsMode(pxConfig *config); static bool toremoveMode(pxConfig *config); -static bool torestoreMode(pxConfig *config); static bool torevertMode(pxConfig *config); +static bool completedrevertMode(pxConfig *config); static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, const char *state); -static bool magicDSRunComplete(pxConfig *config, bool setmagicked); static bool magicDSGetIDs(pxConfig *config, psString stage, psS64 magic_id, psS64 *stage_id, psS64 *cam_id); @@ -70,9 +70,10 @@ MODECASE(MAGICDSTOOL_MODE_TODESTREAK, todestreakMode); MODECASE(MAGICDSTOOL_MODE_ADDDESTREAKEDFILE, adddestreakedfileMode); + MODECASE(MAGICDSTOOL_MODE_ADVANCERUN, advancerunMode); MODECASE(MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE,revertdestreakedfileMode); MODECASE(MAGICDSTOOL_MODE_GETSKYCELLS, getskycellsMode); MODECASE(MAGICDSTOOL_MODE_TOREMOVE, toremoveMode); - MODECASE(MAGICDSTOOL_MODE_TORESTORE, torestoreMode); MODECASE(MAGICDSTOOL_MODE_TOREVERT, torevertMode); + MODECASE(MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertMode); default: psAbort("invalid option (this should not happen)"); @@ -105,5 +106,5 @@ PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false); - PXOPT_LOOKUP_BOOL(re_place, config->args, "-replace", false); + PXOPT_LOOKUP_BOOL(noreplace, config->args, "-noreplace", false); PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); @@ -119,5 +120,5 @@ PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); - PXOPT_COPY_S64(config->args, where, "-magic_id","magic_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id","magicRun.magic_id", "=="); PXOPT_COPY_S32(config->args, where, "-streaks_max","streaks", "<="); @@ -268,5 +269,5 @@ outroot, recoveryroot, - re_place, + noreplace ? 0 :1, // re_place 0); // remove @@ -396,23 +397,28 @@ PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // look for "inputs" that need to processed - psString query = pxDataGet("magicdstool_todestreak.sql"); + psString sql_file = NULL; + psStringAppend(&sql_file, "magicdstool_todestreak_%s.sql", stage); + + psString query = pxDataGet(sql_file); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", sql_file); + psFree(sql_file); + return false; + } + psFree(sql_file); if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } @@ -656,13 +662,4 @@ } - if (!magicDSRunComplete(config, setmagicked)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psDBCommit(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -738,7 +735,13 @@ } -static bool magicDSRunComplete(pxConfig *config, bool setmagicked) +static bool advancerunMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); // look for completed magicDSRuns @@ -749,4 +752,10 @@ } + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -766,4 +775,8 @@ return true; } + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } for (long i = 0; i < psArrayLength(output); i++) { psMetadata *row = output->data[i]; @@ -771,8 +784,12 @@ psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id"); - // if requested, set stageRun.magicked + // if re_place, set stageRun.magicked + bool setmagicked = psMetadataLookupBool(NULL, row, "re_place"); if (setmagicked && !setRunMagicked(config, magic_ds_id)) { psError(PS_ERR_UNKNOWN, false, "failed to change stageRun.magicked for magic_ds_id: %" PRId64, magic_ds_id); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -783,8 +800,14 @@ magic_ds_id); psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } } - + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } return true; @@ -795,4 +818,10 @@ { PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(i_am_sure, config->args, "-i_am_sure", true); + if (!i_am_sure) { + psError(PS_ERR_UNKNOWN, true, "Reverting destreaked files must be done carefully. -i_am_sure is required."); + return false; + } psMetadata *where = psMetadataAlloc(); @@ -802,5 +831,5 @@ pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - psString query = psStringCopy("DELETE FROM magicDSFile USING magicDSFile, magicDSRun WHERE (magicDSRun.magic_ds_id = magicDSFile.magic_ds_id) AND magicDSFile.fault != 0"); + psString query = pxDataGet("magicdstool_revertdestreakedfile.sql"); if (psListLength(where->list)) { @@ -808,4 +837,7 @@ psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search arguments are required"); + return false; } psFree(where); @@ -813,40 +845,44 @@ if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "failed to revert"); - return false; - } - return true; -} - -static bool getskycellsMode(pxConfig *config) -{ - // required - PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); + psFree(query); + return false; + } + psFree(query); + return true; +} + +static bool completedrevertMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); - - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psString query = pxDataGet("magicdstool_getskycells.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psString query = pxDataGet("magicdstool_completedrevert.sql"); + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + psString whereString = NULL; if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&whereString, "\nAND %s", whereClause); psFree(whereClause); } psFree(where); - if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } + if (!p_psDBRunQueryF(config->dbh, query, whereString ? whereString : "")) { + psFree(whereString); + psError(PS_ERR_UNKNOWN, false, "failed to revert"); + return false; + } + psFree(whereString); psFree(query); - psArray *output = p_psDBFetchResult(config->dbh); if (!output) { @@ -868,14 +904,24 @@ return true; } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); + for (int i=0; idata[i]; + psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id"); + psString old_state = psMetadataLookupStr(NULL, row, "state"); + psString new_state; + if (!strcmp(old_state, "goto_censored")) { + new_state = "censored"; + } else if (!strcmp(old_state, "goto_restored")) { + new_state = "restored"; + } else { + psError(PXTOOLS_ERR_PROG, true, "unexpected state found: %s", old_state); psFree(output); return false; } - } - + char *query2 = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query2, new_state, magic_ds_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to set run magicDSRun.state to %s", new_state); + return false; + } + } psFree(output); @@ -883,43 +929,16 @@ } -static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!( - (strncmp(state, "new", 4) == 0) - || (strncmp(state, "full", 5) == 0) - ) - ) { - psError(PS_ERR_UNKNOWN, false, - "invalid magicDSRun state: %s", state); - return false; - } - - char *query = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, state, magic_ds_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for magic_id %" PRId64, magic_ds_id); - return false; - } - - return true; -} - -static bool toremoveMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); +static bool getskycellsMode(pxConfig *config) +{ + // required + PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); - PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // look for "inputs" that need to processed - psString query = pxDataGet("magicdstool_toremove.sql"); + psString query = pxDataGet("magicdstool_getskycells.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -929,17 +948,10 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } psFree(where); - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { + if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -970,5 +982,5 @@ if (psArrayLength(output)) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "toremove", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -981,5 +993,35 @@ return true; } -static bool torestoreMode(pxConfig *config) + +static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!((strcmp(state, "new") == 0) || + (strcmp(state, "full") == 0) || + (strcmp(state, "restored") == 0) || + (strcmp(state, "censored") == 0) || + (strcmp(state, "purged") == 0) || + (strcmp(state, "goto_restored") == 0) || + (strcmp(state, "goto_censored") == 0) || + (strcmp(state, "goto_purged") == 0)) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid magicDSRun state: %s", state); + return false; + } + + char *query = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, magic_ds_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for magic_id %" PRId64, magic_ds_id); + return false; + } + + return true; +} + +static bool toremoveMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -994,5 +1036,5 @@ // look for "inputs" that need to processed - psString query = pxDataGet("magicdstool_torestore.sql"); + psString query = pxDataGet("magicdstool_toremove.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -1043,5 +1085,5 @@ if (psArrayLength(output)) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "torestore", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "toremove", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -1054,5 +1096,4 @@ return true; } - static bool torevertMode(pxConfig *config) Index: branches/eam_branches/20090820/ippTools/src/magicdstool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/magicdstool.h (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/magicdstool.h (revision 25766) @@ -30,9 +30,10 @@ MAGICDSTOOL_MODE_TODESTREAK, MAGICDSTOOL_MODE_ADDDESTREAKEDFILE, + MAGICDSTOOL_MODE_ADVANCERUN, MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE, MAGICDSTOOL_MODE_GETSKYCELLS, MAGICDSTOOL_MODE_TOREMOVE, - MAGICDSTOOL_MODE_TORESTORE, MAGICDSTOOL_MODE_TOREVERT, + MAGICDSTOOL_MODE_COMPLETEDREVERT, } MAGICDStoolMode; Index: branches/eam_branches/20090820/ippTools/src/magicdstoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/magicdstoolConfig.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/magicdstoolConfig.c (revision 25766) @@ -52,5 +52,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-replace", 0, "replace input files with the destreaked versions", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace input files with the destreaked versions", false); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); @@ -75,6 +75,6 @@ psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-outroot", 0, "define output directory (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-replace", 0, "use the simple output format", false); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-remove", 0, "use the simple output format", false); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace the input wit with the destreaked versions", false); +// psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-remove", 0, "use the simple output format", false); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -93,8 +93,8 @@ // -todestreak psMetadata *todestreakArgs = psMetadataAlloc(); + psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-stage", 0, "limit query to stage (required)", NULL); psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); - psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-stage", 0, "limit query to stage", NULL); psMetadataAddU64(todestreakArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(todestreakArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -114,9 +114,10 @@ psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-component", 0, "search by component", NULL); psMetadataAddS16(revertdestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertdestreakedfileArgs, PS_LIST_TAIL, "-i_am_sure", 0, "confirm that you know what you are doing", false); psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); // -getskycells psMetadata *getskycellsArgs = psMetadataAlloc(); - psMetadataAddS64(getskycellsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic de-streak ID (required)", 0); + psMetadataAddS64(getskycellsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic de-streak ID", 0); psMetadataAddStr(getskycellsArgs, PS_LIST_TAIL, "-class_id", 0, "define class identifier", NULL); psMetadataAddStr(getskycellsArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell identifier", NULL); @@ -130,12 +131,4 @@ psMetadataAddU64(toremoveArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(toremoveArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - - // -torestore - psMetadata *torestoreArgs = psMetadataAlloc(); - psMetadataAddS64(torestoreArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); - psMetadataAddS64(torestoreArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); - psMetadataAddStr(torestoreArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); - psMetadataAddU64(torestoreArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(torestoreArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -torevert @@ -147,4 +140,16 @@ psMetadataAddU64(torevertArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(torevertArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -completedrevert + psMetadata *completedrevertArgs = psMetadataAlloc(); + psMetadataAddS64(completedrevertArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); + psMetadataAddStr(completedrevertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddU64(completedrevertArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -advancerun + psMetadata *advancerunArgs = psMetadataAlloc(); + psMetadataAddS64(advancerunArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); + psMetadataAddStr(advancerunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddU64(advancerunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psFree(now); @@ -163,4 +168,6 @@ PXOPT_ADD_MODE("-adddestreakedfile", "add a de-streaked file", MAGICDSTOOL_MODE_ADDDESTREAKEDFILE, adddestreakedfileArgs); + PXOPT_ADD_MODE("-advancerun", "change state for runs that have finished destrreaking", + MAGICDSTOOL_MODE_ADVANCERUN, advancerunArgs); PXOPT_ADD_MODE("-revertdestreakedfile", " revert a faulted de-streaked file", MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE, revertdestreakedfileArgs); @@ -169,8 +176,10 @@ PXOPT_ADD_MODE("-toremove", "backup images pending removal", MAGICDSTOOL_MODE_TOREMOVE, toremoveArgs); - PXOPT_ADD_MODE("-torestore", "images pending restore of exicsed streak pixels", - MAGICDSTOOL_MODE_TORESTORE, torestoreArgs); - PXOPT_ADD_MODE("-torevert", "faulted images to revert", + PXOPT_ADD_MODE("-torevert", "images to restore or revert", MAGICDSTOOL_MODE_TOREVERT, torevertArgs); + + PXOPT_ADD_MODE("-completedrevert", "change state for runs that have finished reverting", + MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertArgs); + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/eam_branches/20090820/ippTools/src/magictool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/magictool.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/magictool.c (revision 25766) @@ -1393,46 +1393,25 @@ } -static bool censorStage(pxConfig *config, psString stage, psString whereClause) -{ - psString queryFile = NULL; - psStringAppend(&queryFile, "magictool_censor_%s.sql", stage); - psString query = pxDataGet(queryFile); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", queryFile); - psFree(queryFile); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(queryFile); - - psStringAppend(&query, " WHERE %s", whereClause); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - return true; -} - static bool censorrunMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); - psError(PS_ERR_PROGRAMMING, true, "-censorrun mode not ready yet"); - return false; - psMetadata *where = psMetadataAlloc(); + + PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", false, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + + if (!magic_id) { + if (!exp_id && !label) { + psError(PS_ERR_UNKNOWN, true, "either -magic_id or exp_id and label is required"); + return false; + } + } // at least one of these required PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); if (!psListLength(where->list)) { @@ -1470,22 +1449,22 @@ // Now queue any destreaked files to be re-verted - // note: on failure censorStage issues the rollback - if (!censorStage(config, "raw", whereClause)) { + // note: on failure magicRestoreStage issues the rollback + if (!magicRestoreStage(config, "raw", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!censorStage(config, "chip", whereClause)) { + if (!magicRestoreStage(config, "chip", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!censorStage(config, "camera", whereClause)) { + if (!magicRestoreStage(config, "camera", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!censorStage(config, "warp", whereClause)) { + if (!magicRestoreStage(config, "warp", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!censorStage(config, "diff", whereClause)) { + if (!magicRestoreStage(config, "diff", whereClause, "goto_censored")) { psFree(whereClause); return false; Index: branches/eam_branches/20090820/ippTools/src/magictoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/magictoolConfig.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/magictoolConfig.c (revision 25766) @@ -165,4 +165,5 @@ psMetadataAddS64(censorrunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID", 0); psMetadataAddS64(censorrunArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0); + psMetadataAddStr(censorrunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); psFree(now); Index: branches/eam_branches/20090820/ippTools/src/pstamptool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pstamptool.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/pstamptool.c (revision 25766) @@ -44,4 +44,5 @@ static bool pendingjobMode(pxConfig *config); static bool updatejobMode(pxConfig *config); +static bool revertjobMode(pxConfig *config); static bool addprojectMode(pxConfig *config); static bool projectMode(pxConfig *config); @@ -80,4 +81,5 @@ MODECASE(PSTAMPTOOL_MODE_PENDINGJOB, pendingjobMode); MODECASE(PSTAMPTOOL_MODE_UPDATEJOB, updatejobMode); + MODECASE(PSTAMPTOOL_MODE_REVERTJOB, revertjobMode); MODECASE(PSTAMPTOOL_MODE_ADDPROJECT, addprojectMode); MODECASE(PSTAMPTOOL_MODE_MODPROJECT, modprojectMode); @@ -224,21 +226,21 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); - // PXOPT_LOOKUP_STR(outFileset, config->args, "-out_fileset", true, false); - PXOPT_LOOKUP_S64(ds_id, config->args, "-ds_id", false, false); - - char *query ="INSERT INTO pstampRequest" - " (state, uri, ds_id, fault)" - " VALUES( 'new', '%s', %" PRId64 ", 0 )"; - if (!p_psDBRunQueryF(config->dbh, query, uri, ds_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psU64 affected = psDBAffectedRows(config->dbh); - if (affected != 1) { - psError(PS_ERR_UNKNOWN, false, - "should have affected one row but %" PRIu64 " rows were modified", - affected); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_S64(ds_id, config->args, "-ds_id", false, false); + + if (!pstampRequestInsert(config->dbh, + 0, // req_id + ds_id, + "new", //state + name, + NULL, // reqType + label, + NULL, // outProduct + uri, + 0 // fault + )) { + psError(PS_ERR_UNKNOWN, false, "failed to insert request"); return false; } @@ -317,4 +319,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-not_req_id", "req_id", "!="); PXOPT_COPY_STR(config->args, where, "-name", "name", "=="); @@ -381,8 +384,5 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = - "SELECT * FROM pstampRequest WHERE state = 'run' AND " - "(SELECT count(*) FROM pstampJob WHERE pstampJob.req_id = pstampRequest.req_id " - " AND pstampJob.state != 'stop') = 0" ; + psString query = pxDataGet("pstamptool_completedreq.sql"); // treat limit == 0 as "no limit" @@ -491,17 +491,40 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); - - // printf("Revert request %" PRId64 "\n", req_id); + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fault", "req_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "pstampRequest.state", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psFree(where); + + // delete any jobs that were queued by requests that didn't complete parsing (pstampRequest.state = 'new' + // If state = 'run' was supplied this will be a no-op + psString query = pxDataGet("pstamptool_revertreq_deletejobs.sql"); + psStringAppend(&query, " AND %s", whereClause); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + // clear fault for requests + query = pxDataGet("pstamptool_revertreq.sql"); + psStringAppend(&query, " AND %s", whereClause); - if (!p_psDBRunQueryF(config->dbh, "DELETE FROM pstampJob where req_id = %" PRId64, req_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!p_psDBRunQueryF(config->dbh, - "UPDATE pstampRequest set state ='new', name=NULL, reqType=NULL, fault=0 where req_id = %" PRId64, req_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); return true; @@ -521,4 +544,5 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + PXOPT_LOOKUP_S64(options, config->args, "-options", false, false); // unless the job is being inserted with stop state require outputBase @@ -545,5 +569,6 @@ fault, exp_id, - outputBase + outputBase, + options )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -569,34 +594,30 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", false, false); - PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", false, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fault", "fault", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = NULL; - - if (!req_id && !job_id) { - fprintf(stderr, "either req_id or job_id must be specified\n"); + if (!psListLength(where->list)) { + fprintf(stderr, "search arguments are required\n"); exit (1); } - if (req_id) { - psStringAppend(&query, - "SELECT" - " pstampJob.*, pstampRequest.name, pstampRequest.outProduct" - " FROM pstampJob" - " JOIN pstampRequest USING(req_id)" - " WHERE req_id = %" PRId64, req_id - ); - } else { - psStringAppend(&query, - "SELECT" - " pstampJob.*, pstampRequest.name, pstampRequest.outProduct" - " FROM pstampJob" - " JOIN pstampRequest USING(req_id)" - " WHERE job_id = %" PRId64, job_id - ); - } + psString query = pxDataGet("pstamptool_listjob.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampJob"); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); // treat limit == 0 as "no limit" @@ -729,4 +750,38 @@ psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" PRIu64 " rows were modified", affected); + return false; + } + + return true; +} + +static bool revertjobMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fault", "fault", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id_min", "req_id", ">="); + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psString query = pxDataGet("pstamptool_revertjob.sql"); + if (!psListLength(where->list) && !all) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters or -all are required"); + return false; + } + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampJob"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); return false; } Index: branches/eam_branches/20090820/ippTools/src/pstamptool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/pstamptool.h (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/pstamptool.h (revision 25766) @@ -39,4 +39,5 @@ PSTAMPTOOL_MODE_JOBRESULT, PSTAMPTOOL_MODE_UPDATEJOB, + PSTAMPTOOL_MODE_REVERTJOB, PSTAMPTOOL_MODE_ADDPROJECT, PSTAMPTOOL_MODE_MODPROJECT, Index: branches/eam_branches/20090820/ippTools/src/pstamptoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pstamptoolConfig.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/pstamptoolConfig.c (revision 25766) @@ -63,7 +63,8 @@ // -addreq psMetadata *addreqArgs = psMetadataAlloc(); - psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0, "define request file uri (required)", NULL); - psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-ds_id", 0, "define request ds_id", 0); - // psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-out_fileset", 0, "define request output_fileset", NULL); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0, "define request file uri (required)", NULL); + psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-ds_id", 0, "define request ds_id", 0); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-name", 0, "define request name", NULL); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-label", 0, "define request label", NULL); // -pendingreq @@ -77,6 +78,7 @@ psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-req_id", 0, "list by req_id", 0); psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-name", 0, "list by name", NULL); + psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-not_req_id", 0, "req_id to not list", 0); psMetadataAddU64(listreqArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(listreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(listreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -completedreq @@ -97,22 +99,26 @@ // -revertreq psMetadata *revertreqArgs = psMetadataAlloc(); - psMetadataAddS64(revertreqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id for which to revert", 0); + psMetadataAddS64(revertreqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id to revert", 0); + psMetadataAddS16(revertreqArgs, PS_LIST_TAIL, "-fault", 0, "fault to revert", 0); + psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-state", 0, "state to revert", NULL); // -addjob psMetadata *addjobArgs = psMetadataAlloc(); - psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-req_id", 0, "define job req_id", 0); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-rownum", 0, "define job rownum", NULL); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-job_type", 0, "define job job_type", "stamp"); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-outputBase", 0, "define job outputBase", NULL); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-state", 0, "new state", "run"); - psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0, "exposure id", 0); - psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0, "new result", 0); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-req_id", 0, "define job req_id", 0); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-rownum", 0, "define job rownum", NULL); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-job_type", 0, "define job job_type", "stamp"); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-outputBase", 0, "define job outputBase", NULL); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-state", 0, "define state", "run"); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure id", 0); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-options", 0, "define options", 0); + psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0, "define job result", 0); // -listjob psMetadata *listjobArgs = psMetadataAlloc(); - psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-req_id", 0, "define request", 0); - psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-job_id", 0, "define job", 0); - psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(listjobArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-req_id", 0, "select by request ID", 0); + psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-job_id", 0, "select by job ID", 0); + psMetadataAddS16(listjobArgs, PS_LIST_TAIL, "-fault", 0, "select by fault", 0); + psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listjobArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -pendingjob @@ -128,4 +134,13 @@ psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-fault", 0, "new result", NULL); + + // -revertjob + psMetadata *revertjobArgs = psMetadataAlloc(); + psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-req_id", 0, "req_id to revert", 0); + psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-req_id_min", 0, "minimum req_id to revert", 0); + psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-job_id", 0, "job_id to revert", 0); + psMetadataAddS16(revertjobArgs, PS_LIST_TAIL, "-fault", 0, "fault to revert", 0); + psMetadataAddBool(revertjobArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted jobs", false); + psMetadataAddU64(revertjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); // -addproject @@ -159,5 +174,5 @@ PXOPT_ADD_MODE("-addreq", "", PSTAMPTOOL_MODE_ADDREQ, addreqArgs); PXOPT_ADD_MODE("-pendingreq", "", PSTAMPTOOL_MODE_PENDINGREQ, pendingreqArgs); - PXOPT_ADD_MODE("-updatereq", "", PSTAMPTOOL_MODE_UPDATEREQ, updatereqArgs); + PXOPT_ADD_MODE("-updatereq", "", PSTAMPTOOL_MODE_UPDATEREQ, updatereqArgs); PXOPT_ADD_MODE("-listreq", "", PSTAMPTOOL_MODE_LISTREQ, listreqArgs); PXOPT_ADD_MODE("-completedreq", "", PSTAMPTOOL_MODE_COMPLETEDREQ, completedreqArgs); @@ -167,5 +182,6 @@ PXOPT_ADD_MODE("-listjob", "", PSTAMPTOOL_MODE_LISTJOB, listjobArgs); PXOPT_ADD_MODE("-pendingjob", "", PSTAMPTOOL_MODE_PENDINGJOB, pendingjobArgs); - PXOPT_ADD_MODE("-updatejob", "", PSTAMPTOOL_MODE_UPDATEJOB, updatejobArgs); + PXOPT_ADD_MODE("-updatejob", "", PSTAMPTOOL_MODE_UPDATEJOB, updatejobArgs); + PXOPT_ADD_MODE("-revertjob", "", PSTAMPTOOL_MODE_REVERTJOB, revertjobArgs); PXOPT_ADD_MODE("-adddatastore", "", PSTAMPTOOL_MODE_ADDDATASTORE, adddatastoreArgs); Index: branches/eam_branches/20090820/ippTools/src/pubtool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pubtool.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/pubtool.c (revision 25766) @@ -259,7 +259,9 @@ // required PXOPT_LOOKUP_S64(pub_id, config->args, "-pub_id", true, false); - PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); // optional + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); PXOPT_LOOKUP_S32(fault, config->args, "-fault", false, false); @@ -269,5 +271,5 @@ } - if (!publishDoneInsert(config->dbh, pub_id, path_base, fault)) { + if (!publishDoneInsert(config->dbh, pub_id, path_base, hostname, dtime_script, fault)) { psError(PS_ERR_UNKNOWN, false, "Unable to add file"); if (!psDBRollback(config->dbh)) { Index: branches/eam_branches/20090820/ippTools/src/pubtoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pubtoolConfig.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/pubtoolConfig.c (revision 25766) @@ -68,4 +68,6 @@ psMetadataAddS64(addArgs, PS_LIST_TAIL, "-pub_id", 0, "define pub_id (required)", 0); psMetadataAddStr(addArgs, PS_LIST_TAIL, "-path_base", 0, "define path_base (required)", NULL); + psMetadataAddStr(addArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL); + psMetadataAddF32(addArgs, PS_LIST_TAIL, "-dtime_script", 0, "define time for script", NAN); psMetadataAddS32(addArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); Index: branches/eam_branches/20090820/ippTools/src/pxadd.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxadd.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxadd.c (revision 25766) @@ -0,0 +1,262 @@ +/* + * pxadd.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxadd.h" + +bool pxaddSetSearchArgs (psMetadata *md) { + + psMetadataAddS64(md, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(md, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + +/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); */ +/* psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); */ +/* psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min", 0, "define min RA (degrees) ", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max", 0, "define max RA (degrees) ", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min", 0, "define min DEC (degrees)", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max", 0, "define max DEC (degrees)", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min", 0, "define min exposure time", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max", 0, "define max exposure time", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min", 0, "define min background", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max", 0, "define max background", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min", 0, "define min background standard deviation", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max background standard deviation", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define min background mean standard deviation (across imfiles)", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max background mean standard deviation (across imfiles)", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min", 0, "define min altitude", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max", 0, "define max altitude", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-az_min", 0, "define min azimuth ", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-az_max", 0, "define max azimuth ", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); */ + + return true; +} + +bool pxaddGetSearchArgs (pxConfig *config, psMetadata *where) { + + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); +/* PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); */ +/* PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); */ +/* PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); */ +/* PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); */ +/* PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); */ +/* PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); */ +/* PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); */ +/* PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); */ +/* PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); */ +/* PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); */ +/* PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); */ +/* PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); */ +/* PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); */ +/* PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); */ +/* PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); */ +/* PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); */ +/* PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); */ + + return true; +} + +bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state, psS64 magicked) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid camRun state: %s", state); + return false; + } + + char *query = "UPDATE addRun SET state = '%s', magicked = %" PRId64 " WHERE add_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, magicked, add_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for add_id %" PRId64, add_id); + return false; + } + + return true; +} + + +bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid chipRun state: %s", state); + return false; + } + + psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET addRun.state = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQueryF(config->dbh, query, state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxaddRunSetLabel(pxConfig *config, psS64 add_id, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + char *query = "UPDATE addRun SET addRun.label = '%s' WHERE add_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, label, add_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for add_id %" PRId64, add_id); + return false; + } + + return true; +} + +bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET addRun.label = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQueryF(config->dbh, query, label)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + +// Need to think more about this to see what we want it to do. BROKEN +bool pxaddQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *label, + char *recipe, + char *dvodb) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("addtool_queue_cam_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + } + // fprintf(stderr,"%s",query); + // queue the exp + // XXX chip_id is being cast here work around psS64 have a different type + // different on 32/64 + if (!p_psDBRunQueryF(config->dbh, query, + "new", // state + workdir ? workdir : "NULL", + "dirty", //workdir_state + label ? label : "NULL", + dvodb ? dvodb : "NULL", + (long long)cam_id + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row"); + return false; + } + + return true; +} Index: branches/eam_branches/20090820/ippTools/src/pxadd.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxadd.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxadd.h (revision 25766) @@ -0,0 +1,44 @@ +/* + * pxadd.h + * + * Copyright (C) 2009 Joshua Hoblitt, Chris Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXADD_H +#define PXADD_H 1 + +#include + +#include "pxtools.h" + +bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state, psS64 magicked); +bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); +bool pxaddRunSetLabel(pxConfig *config, psS64 add_id, const char *label); +bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); + +bool pxaddSetSearchArgs (psMetadata *md); +bool pxaddGetSearchArgs (pxConfig *config, psMetadata *where); + +// Likely BROKEN +bool pxaddQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *label, + char *recipe, + char *dvodb); + + +#endif // PXADD_H Index: branches/eam_branches/20090820/ippTools/src/pxinject.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxinject.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/pxinject.c (revision 25766) @@ -137,7 +137,9 @@ PXOPT_LOOKUP_STR(tmp_class_id, config->args, "-tmp_class_id", true, false); PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false); + PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); // insert with error flag state set to 0 (no errors) - if (!newImfileInsert(config->dbh, exp_id, tmp_class_id, uri, NULL)) { + if (!newImfileInsert(config->dbh, exp_id, tmp_class_id, uri, NULL, bytes, md5sum)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; Index: branches/eam_branches/20090820/ippTools/src/pxinjectConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxinjectConfig.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/pxinjectConfig.c (revision 25766) @@ -61,4 +61,6 @@ psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-tmp_class_id", 0, "define the class ID (required)", NULL); psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-uri", 0, "define the URI (required)", NULL); + psMetadataAddS32(newImfileArgs, PS_LIST_TAIL, "-bytes", 0, "define the size of the file", 0); + psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-md5sum", 0, "define the size of the file", NULL); // -updatenewExp Index: branches/eam_branches/20090820/ippTools/src/pxmagic.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxmagic.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxmagic.c (revision 25766) @@ -0,0 +1,67 @@ +/* + * pxmagic.c + * + * Copyright (C) 2009 IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxmagic.h" + +bool magicRestoreStage(pxConfig *config, psString stage, psString whereClause, psString newState) +{ + psString queryFile = NULL; + psStringAppend(&queryFile, "magictool_restore_%s.sql", stage); + psString query_temp = pxDataGet(queryFile); + if (!query_temp) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", queryFile); + psFree(queryFile); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(queryFile); + + // psStringSubstittute fails on strings created by pxDataGet() + psString query = psStringCopy(query_temp); + // change the @NEW_STATE@ in the sql file to our state + if (!psStringSubstitute(&query, newState, "@NEW_STATE@")) { + psError(PS_ERR_UNKNOWN, false, "failed to substitute state string"); + return false; + } + + psStringAppend(&query, " AND %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + return true; +} Index: branches/eam_branches/20090820/ippTools/src/pxmagic.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxmagic.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxmagic.h (revision 25766) @@ -0,0 +1,29 @@ +/* + * pxmagic.h + * + * Copyright (C) 2009 IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXMAGIC_H +#define PXMAGIC_H 1 + +#include + +#include "pxtools.h" + +extern bool magicRestoreStage(pxConfig *config, psString stage, psString whereClause, psString newState); + +#endif // PXMAGIC_H Index: branches/eam_branches/20090820/ippTools/src/pxtools.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxtools.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/pxtools.c (revision 25766) @@ -37,4 +37,6 @@ if (!strcmp(state, "goto_cleaned")) return true; if (!strcmp(state, "error_cleaned")) return true; + if (!strcmp(state, "goto_purged")) return true; + if (!strcmp(state, "error_purged")) return true; if (!strcmp(state, "goto_scrubbed")) return true; if (!strcmp(state, "error_scrubbed")) return true; @@ -42,12 +44,11 @@ if (!strcmp(state, "update")) return true; if (!strcmp(state, "purged")) return true; - if (!strcmp(state, "goto_purged")) return true; - if (!strcmp(state, "error_purged")) return true; - + if (!strcmp(state, "scrubbed")) return true; return false; } -// 'scrubbed' is a virtual state equivalent to cleaned, but allows files to be removed -// even if the config files is missing +// 'scrubbed' is no longer a virtual state equivalent to cleaned, but allows files to be removed +// even if the config files is missing. This change was prompted as files that are cleaned can +// be regenerated, but that is not certain after being scrubbed. Index: branches/eam_branches/20090820/ippTools/src/pxtools.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxtools.h (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/pxtools.h (revision 25766) @@ -37,4 +37,5 @@ #include "pxtoolsErrorCodes.h" +#include "pxadd.h" #include "pxcam.h" #include "pxchip.h" @@ -45,4 +46,5 @@ #include "pxtag.h" #include "pxtree.h" +#include "pxmagic.h" # define MAX_ROWS 10e9 Index: branches/eam_branches/20090820/ippTools/src/pztool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pztool.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/pztool.c (revision 25766) @@ -401,4 +401,6 @@ PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); PXOPT_LOOKUP_BOOL(row_lock, config->args, "-row_lock", false); + PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false); + PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); // default values @@ -457,5 +459,7 @@ fault, NULL, // epoch - hostname + hostname, + bytes, + md5sum )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -595,5 +599,7 @@ " pzDownloadImfile.class_id," // tmp_class_id " pzDownloadImfile.uri," // uri - " NULL" // epoch + " NULL," // epoch + " pzDownloadImfile.bytes," // bytes + " pzDownloadImfile.md5sum" // md5sum " FROM pzDownloadImfile" " WHERE" Index: branches/eam_branches/20090820/ippTools/src/pztoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pztoolConfig.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/pztoolConfig.c (revision 25766) @@ -95,4 +95,6 @@ psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-label", 0, "define the label for the chip stage", NULL); psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-hostname", 0, "define the host that copied the image", NULL); + psMetadataAddS32(copydoneArgs, PS_LIST_TAIL, "-bytes", 0, "define the size in bytes for the copied image", 0); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-md5sum", 0, "define the md5 sum for the copied image", NULL); psMetadataAddS16(copydoneArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); psMetadataAddBool(copydoneArgs, PS_LIST_TAIL, "-row_lock", 0, "lock pzDownImfile rows while advancing an exposure", false); Index: branches/eam_branches/20090820/ippTools/src/regtool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/regtool.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/regtool.c (revision 25766) @@ -224,4 +224,5 @@ PXOPT_LOOKUP_F32(teltemp_extra, config->args, "-teltemp_extra", false, false); PXOPT_LOOKUP_F32(pon_time, config->args, "-pon_time", false, false); + PXOPT_LOOKUP_S16(burntool_state, config->args, "-burntool_state", false, false); PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); @@ -238,4 +239,6 @@ PXOPT_LOOKUP_TIME(dateobs, config->args, "-dateobs", false, false); PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false); + PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); @@ -307,5 +310,8 @@ quality, NULL, - 0 + 0, + bytes, + md5sum, + 0 // burntool_state )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -327,8 +333,15 @@ PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_S64(config->args, where, "-magicked", "magicked", "=="); + + PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(ordered_by_date, config->args, "-ordered_by_date", false); psString query = pxDataGet("regtool_processedimfile.sql"); @@ -352,4 +365,25 @@ // don't list faulted rows psStringAppend(&query, " %s", "AND rawImfile.fault = 0"); + } + + if (not_destreaked) { + if (destreaked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); + return false; + } + if (magicked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); + return false; + } + psStringAppend(&query, " AND rawImfile.magicked = 0"); + } + if (destreaked) { + psStringAppend(&query, " AND rawImfile.magicked != 0"); + } + + + // add the ORDER BY statement if desired + if (ordered_by_date) { + psStringAppend(&query, " ORDER BY dateobs"); } @@ -456,12 +490,13 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); - - if ((fault == INT16_MAX) && !isfinite(user_1)) { - psError(PS_ERR_UNKNOWN, false, "one of -fault or -user_1 must be selected"); - return false; - } - if ((fault != INT16_MAX) && isfinite(user_1)) { - psError(PS_ERR_UNKNOWN, false, "only one of -fault or -user_1 must be selected"); +/* PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); */ + PXOPT_LOOKUP_S16(burntool_state, config->args, "-burntool_state", false, false); + + if ((fault == INT16_MAX) && !isfinite(burntool_state)) { + psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state must be selected"); + return false; + } + if ((fault != INT16_MAX) && isfinite(burntool_state)) { + psError(PS_ERR_UNKNOWN, false, "only one of -fault or -burntool_state must be selected"); return false; } @@ -480,5 +515,5 @@ } - if (isfinite(user_1)) { + if (isfinite(burntool_state)) { psString query = pxDataGet("regtool_updateprocessedimfile.sql"); if (!query) { @@ -487,5 +522,5 @@ } - if (!p_psDBRunQueryF(config->dbh, query, user_1, exp_id, class_id)) { + if (!p_psDBRunQueryF(config->dbh, query, burntool_state, exp_id, class_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); Index: branches/eam_branches/20090820/ippTools/src/regtoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/regtoolConfig.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/regtoolConfig.c (revision 25766) @@ -125,4 +125,7 @@ ADD_OPT(Time, addprocessedimfileArgs, "-dateobs", "define observation time", NULL); ADD_OPT(Str, addprocessedimfileArgs, "-hostname", "define host name", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-md5sum", "define md5sum", NULL); + ADD_OPT(S32, addprocessedimfileArgs, "-bytes", "define bytes", 0); + ADD_OPT(S16, addprocessedimfileArgs, "-burntool_state", "set burntool state", 0); ADD_OPT(S16, addprocessedimfileArgs, "-fault", "set fault code", 0); ADD_OPT(S16, addprocessedimfileArgs, "-quality", "set quality flag", 0); @@ -133,6 +136,10 @@ ADD_OPT(Str, processedimfileArgs, "-exp_name", "search by exposure name", NULL); ADD_OPT(Str, processedimfileArgs, "-class_id", "search by class ID", NULL); + ADD_OPT(Str, processedimfileArgs, "-filter", "search by filter", NULL); ADD_OPT(Time, processedimfileArgs, "-dateobs_begin", "search for exposures by time (>=)", NULL); ADD_OPT(Time, processedimfileArgs, "-dateobs_end", "search for exposures by time (<)", NULL); + ADD_OPT(S64, processedimfileArgs, "-magicked", "search by magicked value", 0); + ADD_OPT(Bool, processedimfileArgs, "-destreaked", "only return imfiles that have been destreaked", false); + ADD_OPT(Bool, processedimfileArgs, "-not_destreaked", "only return imfiles that have not been destreaked", false); ADD_OPT(U64, processedimfileArgs, "-limit", "limit result set to N items", 0); ADD_OPT(Bool, processedimfileArgs, "-faulted", "only return imfiles with a fault status set", false); @@ -153,5 +160,5 @@ ADD_OPT(S64, updateprocessedimfileArgs, "-exp_id", "search by exposure ID", 0); ADD_OPT(Str, updateprocessedimfileArgs, "-class_id", "search by class ID", NULL); - ADD_OPT(F64, updateprocessedimfileArgs, "-user_1", "set user stat (1)", NAN); + ADD_OPT(S16, updateprocessedimfileArgs, "-burntool_state", "set burntool state", 0); ADD_OPT(S16, updateprocessedimfileArgs, "-fault", "set fault code", INT16_MAX); Index: branches/eam_branches/20090820/ippTools/src/stacktool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/stacktool.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/stacktool.c (revision 25766) @@ -139,4 +139,10 @@ PXOPT_COPY_F32(config->args, where, "-select_fwhm_minor_min", "camProcessedExp.fwhm_minor", ">="); PXOPT_COPY_F32(config->args, where, "-select_fwhm_minor_max", "camProcessedExp.fwhm_minor", "<="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m2_max", "camProcessedExp.iq_m2", "<="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m2_min", "camProcessedExp.iq_m2", ">="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m3_max", "camProcessedExp.iq_m3", "<="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m4_min", "camProcessedExp.iq_m4", ">="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m4_max", "camProcessedExp.iq_m4", "<="); + PXOPT_COPY_STR(config->args, where, "-select_exp_type", "rawExp.exp_type", "=="); PXOPT_COPY_F32(config->args, where, "-select_good_frac_min", "warpSkyfile.good_frac", ">="); @@ -853,10 +859,17 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "=="); PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "=="); + +// The following three selectors are incompatible with the sql so omit them +// PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); +// PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); +// PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -873,5 +886,9 @@ psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); - } + } else if (!all) { + psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); + return false; + } + psFree(where); Index: branches/eam_branches/20090820/ippTools/src/stacktoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/stacktoolConfig.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/stacktoolConfig.c (revision 25766) @@ -77,4 +77,10 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_fwhm_minor_min", 0, "define min fwhm (minor axis)", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_fwhm_minor_max", 0, "define max fwhm (minor axis)", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m2_min", 0, "define min iq_m2", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m2_max", 0, "define max iq_m2", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m3_min", 0, "define min iq_m3", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m3_max", 0, "define max iq_m3", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m4_min", 0, "define min iq_m4", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m4_max", 0, "define max iq_m4", NAN); psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random", 0, "use this number of random elements", 0); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); @@ -158,11 +164,18 @@ psMetadata *sumskyfileArgs= psMetadataAlloc(); psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); - psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", 0); psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", 0); +#ifdef notdef + // These don't work so omit (for now) We probably should create a different mode for this type of search. + psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", NULL); +#endif + psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by stackRun.label", NULL); + psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", NULL); + psMetadataAddS16(sumskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddU64(sumskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(sumskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(sumskyfileArgs, PS_LIST_TAIL, "-all", 0, "enable search without arguments", false); // -revertsumskyfile Index: branches/eam_branches/20090820/ippTools/src/warptool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/warptool.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/warptool.c (revision 25766) @@ -53,4 +53,5 @@ static bool tocleanedskyfileMode(pxConfig *config); static bool topurgedskyfileMode(pxConfig *config); +static bool toscrubbedskyfileMode(pxConfig *config); static bool tofullskyfileMode(pxConfig *config); static bool updateskyfileMode(pxConfig *config); @@ -102,4 +103,5 @@ MODECASE(WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileMode); MODECASE(WARPTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileMode); + MODECASE(WARPTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileMode); MODECASE(WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileMode); MODECASE(WARPTOOL_MODE_UPDATESKYFILE, updateskyfileMode); @@ -1196,4 +1198,15 @@ PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_S64(config->args, where, "-magicked", "warpSkyfile.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); + + PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -1211,6 +1224,24 @@ psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); + } else if (!all) { + psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); + return false; } psFree(where); + + if (not_destreaked) { + if (destreaked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); + return false; + } + if (magicked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); + return false; + } + psStringAppend(&query, " AND warpSkyfile.magicked = 0"); + } + if (destreaked) { + psStringAppend(&query, " AND warpSkyfile.magicked != 0"); + } // treat limit == 0 as "no limit" @@ -1668,4 +1699,8 @@ return change_skyfile_data_state(config, "purged", "goto_purged"); } +static bool toscrubbedskyfileMode(pxConfig *config) +{ + return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed"); +} static bool updateskyfileMode(pxConfig *config) Index: branches/eam_branches/20090820/ippTools/src/warptool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/warptool.h (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/warptool.h (revision 25766) @@ -49,4 +49,5 @@ WARPTOOL_MODE_TOCLEANEDSKYFILE, WARPTOOL_MODE_TOPURGEDSKYFILE, + WARPTOOL_MODE_TOSCRUBBEDSKYFILE, WARPTOOL_MODE_TOFULLSKYFILE, WARPTOOL_MODE_UPDATESKYFILE, Index: branches/eam_branches/20090820/ippTools/src/warptoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/warptoolConfig.c (revision 25206) +++ branches/eam_branches/20090820/ippTools/src/warptoolConfig.c (revision 25766) @@ -215,4 +215,14 @@ psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-fake_id", 0, "search by phase 3 version of exposure tag", 0); + psMetadataAddTime(warpedArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(warpedArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-filter", 0, "search for exposures by filter", NULL); + psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-destreaked", 0, "search for destreaked images", false); + psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for images that have not been destreaked", false); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-magicked", 0, "search by magicked value", 0); + psMetadataAddS16(warpedArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + + psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); psMetadataAddU64(warpedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -278,4 +288,9 @@ psMetadataAddS64(topurgedskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); psMetadataAddStr(topurgedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + + // -toscrubbedskyfile + psMetadata *toscrubbedskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); + psMetadataAddStr(toscrubbedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); // -tofullskyfile @@ -328,4 +343,5 @@ PXOPT_ADD_MODE("-tocleanedskyfile", "set skyfile as cleaned", WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileArgs); PXOPT_ADD_MODE("-topurgedskyfile", "set skyfile as purged", WARPTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileArgs); + PXOPT_ADD_MODE("-toscrubbedskyfile", "set skyfile as scrubbed", WARPTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileArgs); PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full (updated)", WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs); PXOPT_ADD_MODE("-updateskyfile", "update fault code for skyfile", WARPTOOL_MODE_UPDATESKYFILE, updateskyfileArgs); Index: branches/eam_branches/20090820/ippconfig/cfh12k/psphot.config =================================================================== --- branches/eam_branches/20090820/ippconfig/cfh12k/psphot.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/cfh12k/psphot.config (revision 25766) @@ -1,36 +1,10 @@ # turn these on to see specific outputs -SAVE.OUTPUT BOOL TRUE -SAVE.BACKMDL BOOL TRUE -SAVE.PSF BOOL TRUE SAVE.PLOTS BOOL TRUE - -BACKGROUND.XBIN S32 128 # size of background superpixels -BACKGROUND.YBIN S32 128 # size of background superpixels - -# image background parameters -IMSTATS_NPIX S32 10000 # number of pixels to use for sky estimate boxes: - -PSF_SN_LIM F32 50 # minimum S/N for stars used for PSF model -PSF_MAX_NSTARS S32 300 # limit number of stars used for PSF model - -PEAKS_NSIGMA_LIMIT F32 25.0 # peak significance threshold -PEAKS_NSIGMA_LIMIT_2 F32 10.0 # peak significance threshold # PSF model parameters : choose the PSF model desired PSF_MODEL STR PS_MODEL_QGAUSS -MOMENTS_SN_MIN F32 20.0 # min S/N to measure moments -EXT_MIN_SN F32 50.0 # fit galaxies above this S/N limit -FULL_FIT_SN_LIM F32 50.0 -AP_MIN_SN F32 25.0 -MEASURE.APTREND BOOL FALSE - -# BREAK_POINT may be one of (PEAKS, MOMENTS, PSFMODEL, ENSEMBLE) -# BREAK_POINT STR ENSEMBLE - -OUTPUT.FORMAT STR PS1_DEV_1 - -# masking parameters (XXX EAM : rework this to use psRegion like ANALYSIS_REGION) +# XXX are these needed? XMIN F32 +10 # minimum valid x-coord XMAX F32 -10 # maximum valid x-coord @@ -38,3 +12,2 @@ YMAX F32 -10 # maximum valid y-coord -USE_FOOTPRINTS BOOL TRUE # use new pmFootprint peak packaging Index: branches/eam_branches/20090820/ippconfig/esowfi/psphot.config =================================================================== --- branches/eam_branches/20090820/ippconfig/esowfi/psphot.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/esowfi/psphot.config (revision 25766) @@ -1,30 +1,3 @@ -SAVE.BACKMDL BOOL TRUE -SAVE.BACKMDL.STDEV BOOL TRUE -SAVE.BACKGND BOOL TRUE -SAVE.BACKSUB BOOL TRUE -SAVE.PLOTS BOOL FALSE -SAVE.RESID BOOL TRUE -SAVE.PSF BOOL TRUE -LOAD.PSF BOOL FALSE - -IMSTATS_NPIX S32 3000 # number of pixels to use for sky estimate boxes: - -#PSPHOT_TEST METADATA -# SAVE.BACKSUB BOOL FALSE -# SAVE.RESID BOOL TRUE -#END -# -#PEAKS_OUTPUT_FILE STR peaks.dat -#MOMENTS_OUTPUT_FILE STR moments.dat - -#PSF_SN_LIM F32 20.0 # minimum S/N for stars used for PSF model -MOMENTS_SN_MIN F32 20.0 # min S/N to measure moments -FULL_FIT_SN_LIM F32 100.0 -EXT_MIN_SN F32 100.0 # fit galaxies above this S/N limit - -PEAKS_NSIGMA_LIMIT F32 10.0 # peak significance threshold -BREAK_POINT STR ENSEMBLE # limit total processing for now (for speed) - - -SKY_STAT STR ROBUST_MEDIAN # statistic used to measure background +# choose the PSF model +PSF_MODEL STR PS_MODEL_QGAUSS Index: branches/eam_branches/20090820/ippconfig/gpc1/Makefile.am =================================================================== --- branches/eam_branches/20090820/ippconfig/gpc1/Makefile.am (revision 25206) +++ branches/eam_branches/20090820/ippconfig/gpc1/Makefile.am (revision 25766) @@ -20,5 +20,4 @@ psastro.config \ pswarp.config \ - ppStack.config \ rejections.config Index: branches/eam_branches/20090820/ippconfig/gpc1/camera.config =================================================================== --- branches/eam_branches/20090820/ippconfig/gpc1/camera.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/gpc1/camera.config (revision 25766) @@ -132,4 +132,5 @@ CMF.XSRC STR {CHIP.NAME}.xsrc # use .PSF and .EXT? CMF.XFIT STR {CHIP.NAME}.xfit # use .PSF and .EXT? + CMF.DETEFF STR {CHIP.NAME}.deteff PSF.HEAD STR {CHIP.NAME}.hdr @@ -146,2 +147,4 @@ PHOTCODE.RULE STR {DETECTOR}.{FILTER.ID}.{CHIP.NAME} # Rule for generating photcode + +BURNTOOL.STATE.GOOD S16 13 # Value for burntool_state with the most recent bt version. Index: branches/eam_branches/20090820/ippconfig/gpc1/format_20080925.config =================================================================== --- branches/eam_branches/20090820/ippconfig/gpc1/format_20080925.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/gpc1/format_20080925.config (revision 25766) @@ -222,4 +222,6 @@ FPA.TELTEMP.TRUSS STR TELTEMTR # Mid truss temperatures (C FPA.TELTEMP.EXTRA STR TELTEMEX # Miscellaneous temperatures (C) + + FPA.BURNTOOL.APPLIED STR BTOOLAPP FPA.PON.TIME STR PONTIME Index: branches/eam_branches/20090820/ippconfig/gpc1/format_20080929.config =================================================================== --- branches/eam_branches/20090820/ippconfig/gpc1/format_20080929.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/gpc1/format_20080929.config (revision 25766) @@ -217,4 +217,6 @@ FPA.TELTEMP.TRUSS STR TELTEMTR # Mid truss temperatures (C FPA.TELTEMP.EXTRA STR TELTEMEX # Miscellaneous temperatures (C) + + FPA.BURNTOOL.APPLIED STR BTOOLAPP FPA.PON.TIME STR PONTIME Index: branches/eam_branches/20090820/ippconfig/gpc1/format_20081011.config =================================================================== --- branches/eam_branches/20090820/ippconfig/gpc1/format_20081011.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/gpc1/format_20081011.config (revision 25766) @@ -208,4 +208,6 @@ FPA.TELTEMP.TRUSS STR TELTEMTR # Mid truss temperatures (C FPA.TELTEMP.EXTRA STR TELTEMEX # Miscellaneous temperatures (C) + + FPA.BURNTOOL.APPLIED STR BTOOLAPP FPA.PON.TIME STR PONTIME Index: branches/eam_branches/20090820/ippconfig/gpc1/format_20090120.config =================================================================== --- branches/eam_branches/20090820/ippconfig/gpc1/format_20090120.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/gpc1/format_20090120.config (revision 25766) @@ -207,4 +207,6 @@ FPA.TELTEMP.TRUSS STR TELTEMTR # Mid truss temperatures (C FPA.TELTEMP.EXTRA STR TELTEMEX # Miscellaneous temperatures (C) + + FPA.BURNTOOL.APPLIED STR BTOOLAPP FPA.PON.TIME STR PONTIME Index: branches/eam_branches/20090820/ippconfig/gpc1/format_20090220.config =================================================================== --- branches/eam_branches/20090820/ippconfig/gpc1/format_20090220.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/gpc1/format_20090220.config (revision 25766) @@ -185,5 +185,5 @@ FPA.ALT STR ALT FPA.AZ STR AZ - FPA.TEMP STR DETTEM + # FPA.TEMP STR DETTEM FPA.M1X STR M1X FPA.M1Y STR M1Y @@ -207,4 +207,6 @@ FPA.TELTEMP.TRUSS STR TELTEMTR # Mid truss temperatures (C FPA.TELTEMP.EXTRA STR TELTEMEX # Miscellaneous temperatures (C) + + FPA.BURNTOOL.APPLIED STR BTOOLAPP FPA.PON.TIME STR PONTIME @@ -501,4 +503,6 @@ # How to translation PS concepts into database lookups DATABASE METADATA + # this rule is fragile : does not match the camera + FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 END Index: branches/eam_branches/20090820/ippconfig/gpc1/format_orig.config =================================================================== --- branches/eam_branches/20090820/ippconfig/gpc1/format_orig.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/gpc1/format_orig.config (revision 25766) @@ -215,4 +215,6 @@ FPA.TELTEMP.TRUSS STR TELTEMTR # Mid truss temperatures (C FPA.TELTEMP.EXTRA STR TELTEMEX # Miscellaneous temperatures (C) + + FPA.BURNTOOL.APPLIED STR BTOOLAPP FPA.PON.TIME STR PONTIME Index: branches/eam_branches/20090820/ippconfig/gpc1/ppStack.config =================================================================== --- branches/eam_branches/20090820/ippconfig/gpc1/ppStack.config (revision 25206) +++ (revision ) @@ -1,38 +1,0 @@ -# Recipe configuration for ppStack (image combination) - -ITER S32 1 # Number of rejection iterations -COMBINE.REJ F32 4.0 # Rejection threshold in combination (sigma) -MASK.VAL STR MASK.VALUE,CONV.BAD # Mask value of input bad pixels -MASK.BAD STR BLANK # Mask value to give bad pixels -MASK.POOR STR CONV.POOR # Mask value to give poor pixels -POOR.FRACTION F32 0.1 # Maximum fraction of bad weight for poor pixels -THRESHOLD.MASK F32 0.8 # Threshold for mask deconvolution (0..1) -IMAGE.REJ F32 0.2 # Rejected pixel fraction threshold for rejecting entire image -ROWS S32 64 # Number of rows to read at once -VARIANCE BOOL TRUE # Use variance in rejection? -SAFE BOOL FALSE # Play safe when combining small number of values? - -RENORM BOOL FALSE # Renormalise variance maps? -RENORM.MEAN STR ROBUST_MEDIAN # Statistic to use for mean in renormalisation -RENORM.STDEV STR ROBUST_STDEV # Statistic to use for stdev in renormalisation -RENORM.WIDTH S32 300 # Size of renormalisation boxes (pixels) - -SOURCE.RADIUS F32 1.0 # Radius (pixels) for matching sources -SOURCE.MIN S32 15 # Minimum number of sources for merging source lists -SOURCE.ITER S32 2 # Number of rejection iterations for magnitude difference -SOURCE.REJ F32 2.0 # Rejection limit (sigma) for magnitude difference - -PSF.INSTANCES S32 9 # Number of instances for PSF generation -PSF.RADIUS F32 20.0 # Radius for PSF generation -PSF.ORDER S32 2 # Order of spatial variation for PSF generation -PSF.MODEL STR PS_MODEL_GAUSS # Model for PSF generation - -TEMP.IMAGE STR conv.im.fits # Suffix for convolved images -TEMP.MASK STR conv.mk.fits # Suffix for convolved masks -TEMP.VARIANCE STR conv.var.fits # Suffix for convolved variance maps -TEMP.DELETE BOOL FALSE # Delete temporary files on completion? - -PR METADATA - CONVOLVE BOOL FALSE - IMAGE.REJ F32 0.5 -END Index: branches/eam_branches/20090820/ippconfig/gpc1/psphot.config =================================================================== --- branches/eam_branches/20090820/ippconfig/gpc1/psphot.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/gpc1/psphot.config (revision 25766) @@ -1,56 +1,31 @@ -# turn these on to see specific outputs -SAVE.OUTPUT BOOL TRUE -SAVE.BACKMDL BOOL TRUE -SAVE.PSF BOOL TRUE -SAVE.PLOTS BOOL TRUE +BACKGROUND.XBIN S32 256 # size of background superpixels +BACKGROUND.YBIN S32 256 # size of background superpixels -# for testing: -# BREAK_POINT STR BACKMDL +SOFTEN.VARIANCE BOOL TRUE # XXX drop? soften the errors for bright pixels +SOFTEN.VARIANCE.FRACTION F32 0.005 -BACKGROUND.XBIN S32 256 # size of background superpixels -BACKGROUND.YBIN S32 256 # size of background superpixels +PEAKS_SMOOTH_SIGMA F32 2.5 # smoothing kernel sigma in pixels -# image background parameters -IMSTATS_NPIX S32 10000 # number of pixels to use for sky estimate boxes: -SKY_STAT STR FITTED_MEAN_V4 # statistic used to measure background -SKY_CLIP_SIGMA F32 2.0 # statistic used to measure background +PSF_CLUMP_NX S32 3 # subdivide image in to NX x NY regions for +PSF_CLUMP_NY S32 3 # selecting PSF stars +PSF_CLUMP_GRID_SCALE F32 2.5 # XXX too large? size of Sx,Sy image pixel -PEAKS_SMOOTH_SIGMA F32 2.5 # smoothing kernel sigma in pixels -PEAKS_NMAX S32 5000 # on first pass, only keep NMAX peaks (0 == all) -PEAKS_NSIGMA_LIMIT F32 25.0 # peak significance threshold -PEAKS_NSIGMA_LIMIT_2 F32 5.0 # peak significance threshold - -PSF_SN_LIM F32 25 # minimum S/N for stars used for PSF model -PSF_MAX_NSTARS S32 300 # limit number of stars used for PSF model -PSF_CLUMP_NX S32 3 # subdivide image in to NX x NY regions for -PSF_CLUMP_NY S32 3 # selecting PSF stars -PSF_MOMENTS_RADIUS F32 12 # calculate initial source moments with this radius -PSF_CLUMP_GRID_SCALE F32 2.5 # size of Sx,Sy image pixel - -PSF.RESIDUALS BOOL TRUE # generate the residuals? +PSF.RESIDUALS.RADIUS F32 9.0 # keep this pixel if residual is more significant than this +PSF.RESIDUALS.SPATIAL_ORDER S32 1 # fit spatial variations of the residuals at this order (0,1) # PSF model parameters : choose the PSF model desired -PSF_MODEL STR PS_MODEL_PS1_V1 -PSPHOT.CR.NSIGMA.SOFTEN F32 0.0025 # Softening parameter for weights +PSF_MODEL STR PS_MODEL_PS1_V1 +PSPHOT.CR.NSIGMA.SOFTEN F32 0.0025 # Softening parameter for weights in CR stat -PSF.TREND.MODE STR MAP # other options: POLY_CHEB, POLY_ORD PSF.TREND.NX S32 3 PSF.TREND.NY S32 3 -EXT_MIN_SN F32 50.0 # fit galaxies above this S/N limit -FULL_FIT_SN_LIM F32 25.0 -MOMENTS_SN_MIN F32 5.0 -AP_MIN_SN F32 5.0 +# rename NONLINEAR_FIT_SN_LIM? +FULL_FIT_SN_LIM F32 25.0 -OUTPUT.FORMAT STR PS1_V2 +APTREND.ORDER.MAX S32 3 -PSF_SHAPE_NSIGMA F32 3.0 # max significance for shape variation -PSF_MAX_CHI F32 50.0 # reject objects worse that this - -MEASURE.APTREND BOOL TRUE ### XXX for now, skip this (too many errors) -APTREND.ORDER.MAX S32 3 - -DIAGNOSTIC.PLOTS METADATA +DIAGNOSTIC.PLOTS METADATA IMAGE.BACKGROUND.CELL.HISTOGRAM BOOL FALSE IMAGE.BACKGROUND.CELL.HISTOGRAM.X S32 32 @@ -58,8 +33,6 @@ END -USE_FOOTPRINTS BOOL TRUE # use new pmFootprint peak packaging - -DIFF METADATA - BACKGROUND.XBIN S32 32 # size of background superpixels - BACKGROUND.YBIN S32 32 # size of background superpixels +DIFF METADATA + BACKGROUND.XBIN S32 32 # size of background superpixels + BACKGROUND.YBIN S32 32 # size of background superpixels END Index: branches/eam_branches/20090820/ippconfig/isp/psphot.config =================================================================== --- branches/eam_branches/20090820/ippconfig/isp/psphot.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/isp/psphot.config (revision 25766) @@ -1,76 +1,13 @@ # turn these on to see specific outputs -SAVE.BACKMDL BOOL TRUE -SAVE.BACKGND BOOL TRUE -SAVE.BACKSUB BOOL TRUE -SAVE.RESID BOOL TRUE -SAVE.PSF BOOL TRUE SAVE.PLOTS BOOL TRUE -# image statistics parameters -IMSTATS_NPIX S32 5000 # number of pixels to use for sky estimate boxes: -SKY_STAT STR FITTED_MEAN_V4 # statistic used to measure background - -PSF_SN_LIM F32 30 # minimum S/N for stars used for PSF model -PSF_MAX_NSTARS S32 300 # limit number of stars used for PSF model - +# PSF model parameters : choose the PSF model PSF_MODEL STR PS_MODEL_PGAUSS -MOMENTS_SN_MIN F32 30.0 -MOMENTS_AR_MAX F32 2.0 # maximum axial ratio: 1 / AR < (sx / sy) < AR - -#EXT_MIN_SN F32 50.0 # fit galaxies above this S/N limit -#FULL_FIT_SN_LIM F32 50.0 -#AP_MIN_SN F32 20.0 +# xxx check this: PSF_CLUMP_NSIGMA F32 3.5 # region of Sx,Sy plane to use for selecting PSF stars -# PSFTREND must be a 2D polynomial -# the specified values are ignored but define the active components of the polynomial -PSF.TREND.MASK METADATA - NORDER_X S32 3 # number of x orders - NORDER_Y S32 3 # number of y orders - VAL_X00_Y00 F64 1 # polynomial coefficient - - VAL_X01_Y00 F64 1 # polynomial coefficient - VAL_X00_Y01 F64 1 # polynomial coefficient - - VAL_X02_Y00 F64 1 # polynomial coefficient - VAL_X01_Y01 F64 1 # polynomial coefficient - VAL_X00_Y02 F64 1 # polynomial coefficient - - VAL_X03_Y00 F64 1 # polynomial coefficient - VAL_X02_Y01 F64 1 # polynomial coefficient - VAL_X01_Y02 F64 1 # polynomial coefficient - VAL_X00_Y03 F64 1 # polynomial coefficient - NELEMENTS S32 10 # number of unmasked components -END # folder for 4D polynomial - +# XXX set other 2D trends PSF.TREND.NX S32 4 PSF.TREND.NY S32 4 - -XMIN F32 15 - -PSF.RESIDUALS BOOL true -PSF.RESIDUALS.SPATIAL_ORDER S32 1 - -BREAK_POINT STR ENSEMBLE -OUTPUT.FORMAT STR PS1_DEV_0 - -PSPHOT.SUMMIT METADATA - PEAKS_SMOOTH_SIGMA F32 0.8 # peak significance threshold - PEAKS_NSIGMA_LIMIT F32 50.0 # peak significance threshold - PEAKS_NMAX S32 1000 - SAVE.RESID BOOL FALSE - PSF_SN_LIM F32 25 # minimum S/N for stars used for PSF model - MOMENTS_SN_MIN F32 25.0 - PSF_MODEL STR PS_MODEL_PGAUSS -END - -DIAGNOSTIC.PLOTS METADATA - IMAGE.BACKGROUND.CELL.HISTOGRAM BOOL FALSE - IMAGE.BACKGROUND.CELL.HISTOGRAM.X S32 4 - IMAGE.BACKGROUND.CELL.HISTOGRAM.Y S32 7 -END - -SKY_BIAS F32 0.5 -AP_MIN_SN F32 30.0 Index: branches/eam_branches/20090820/ippconfig/lbc_red/psphot.config =================================================================== --- branches/eam_branches/20090820/ippconfig/lbc_red/psphot.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/lbc_red/psphot.config (revision 25766) @@ -1,42 +1,6 @@ # turn these on to see specific outputs -SAVE.OUTPUT BOOL TRUE -SAVE.BACKMDL BOOL TRUE -SAVE.PSF BOOL TRUE SAVE.PLOTS BOOL TRUE - -BACKGROUND.XBIN S32 128 # size of background superpixels -BACKGROUND.YBIN S32 128 # size of background superpixels - -# image background parameters -IMSTATS_NPIX S32 10000 # number of pixels to use for sky estimate boxes: -SKY_STAT STR FITTED_MEAN_V4 # statistic used to measure background -SKY_CLIP_SIGMA F32 2.0 # statistic used to measure background - -PSF_SN_LIM F32 100 # minimum S/N for stars used for PSF model -PSF_MAX_NSTARS S32 300 # limit number of stars used for PSF model # PSF model parameters : choose the PSF model desired PSF_MODEL STR PS_MODEL_QGAUSS - -MOMENTS_SN_MIN F32 30.0 -EXT_MIN_SN F32 50.0 # fit galaxies above this S/N limit -FULL_FIT_SN_LIM F32 50.0 -AP_MIN_SN F32 50.0 - -# OUTPUT.FORMAT STR SMPDATA -OUTPUT.FORMAT STR PS1_DEV_1 - -PSF_SHAPE_NSIGMA F32 3.0 # max significance for shape variation -PSF_MAX_CHI F32 50.0 # reject objects worse that this - -#PSF.TREND.MODE STR MAP -#PSF.TREND.NX S32 1 -#PSF.TREND.NY S32 1 - -DIAGNOSTIC.PLOTS METADATA - IMAGE.BACKGROUND.CELL.HISTOGRAM BOOL FALSE - IMAGE.BACKGROUND.CELL.HISTOGRAM.X S32 -1 - IMAGE.BACKGROUND.CELL.HISTOGRAM.Y S32 12 -END - Index: branches/eam_branches/20090820/ippconfig/lulin/psphot.config =================================================================== --- branches/eam_branches/20090820/ippconfig/lulin/psphot.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/lulin/psphot.config (revision 25766) @@ -1,43 +1,6 @@ # turn these on to see specific outputs -SAVE.OUTPUT BOOL TRUE -SAVE.BACKMDL BOOL TRUE -SAVE.PSF BOOL TRUE SAVE.PLOTS BOOL TRUE - -BACKGROUND.XBIN S32 128 # size of background superpixels -BACKGROUND.YBIN S32 128 # size of background superpixels - -# image background parameters -IMSTATS_NPIX S32 10000 # number of pixels to use for sky estimate boxes: -SKY_STAT STR FITTED_MEAN_V4 # statistic used to measure background -SKY_CLIP_SIGMA F32 2.0 # statistic used to measure background - -PSF_SN_LIM F32 100 # minimum S/N for stars used for PSF model -PSF_MAX_NSTARS S32 300 # limit number of stars used for PSF model # PSF model parameters : choose the PSF model desired PSF_MODEL STR PS_MODEL_QGAUSS - -MOMENTS_SN_MIN F32 30.0 -EXT_MIN_SN F32 50.0 # fit galaxies above this S/N limit -FULL_FIT_SN_LIM F32 50.0 -AP_MIN_SN F32 50.0 - -# OUTPUT.FORMAT STR SMPDATA -OUTPUT.FORMAT STR PS1_DEV_1 - -PSF_SHAPE_NSIGMA F32 3.0 # max significance for shape variation -PSF_MAX_CHI F32 50.0 # reject objects worse that this - -#PSF.TREND.MODE STR MAP -#PSF.TREND.NX S32 1 -#PSF.TREND.NY S32 1 - -DIAGNOSTIC.PLOTS METADATA - IMAGE.BACKGROUND.CELL.HISTOGRAM BOOL FALSE - IMAGE.BACKGROUND.CELL.HISTOGRAM.X S32 -1 - IMAGE.BACKGROUND.CELL.HISTOGRAM.Y S32 12 -END - -USE_FOOTPRINTS BOOL TRUE # use new pmFootprint peak packaging Index: branches/eam_branches/20090820/ippconfig/megacam/psphot.config =================================================================== --- branches/eam_branches/20090820/ippconfig/megacam/psphot.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/megacam/psphot.config (revision 25766) @@ -1,44 +1,10 @@ # turn these on to see specific outputs -SAVE.OUTPUT BOOL TRUE -SAVE.BACKMDL BOOL TRUE -SAVE.PSF BOOL TRUE SAVE.PLOTS BOOL TRUE - -BACKGROUND.XBIN S32 128 # size of background superpixels -BACKGROUND.YBIN S32 128 # size of background superpixels - -# image background parameters -IMSTATS_NPIX S32 10000 # number of pixels to use for sky estimate boxes: -SKY_STAT STR FITTED_MEAN_V4 # statistic used to measure background -SKY_CLIP_SIGMA F32 2.0 # statistic used to measure background - -PSF_SN_LIM F32 20 # minimum S/N for stars used for PSF model -PSF_MAX_NSTARS S32 300 # limit number of stars used for PSF model # PSF model parameters : choose the PSF model desired PSF_MODEL STR PS_MODEL_QGAUSS -PSF_MOMENTS_RADIUS F32 10.0 # calculate initial source moments with this radius - -MOMENTS_SN_MIN F32 2.0 -EXT_MIN_SN F32 50.0 # fit galaxies above this S/N limit -FULL_FIT_SN_LIM F32 10.0 -AP_MIN_SN F32 2.0 - -PSF_SHAPE_NSIGMA F32 3.0 # max significance for shape variation -PSF_MAX_CHI F32 50.0 # reject objects worse that this - PSF.TREND.MODE STR MAP PSF.TREND.NX S32 1 PSF.TREND.NY S32 1 - -DIAGNOSTIC.PLOTS METADATA - IMAGE.BACKGROUND.CELL.HISTOGRAM BOOL FALSE - IMAGE.BACKGROUND.CELL.HISTOGRAM.X S32 -1 - IMAGE.BACKGROUND.CELL.HISTOGRAM.Y S32 12 -END - -USE_FOOTPRINTS BOOL TRUE # use new pmFootprint peak packaging - -PEAKS_NSIGMA_LIMIT F32 10.0 Index: branches/eam_branches/20090820/ippconfig/mosaic2/camera.config =================================================================== --- branches/eam_branches/20090820/ippconfig/mosaic2/camera.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/mosaic2/camera.config (revision 25766) @@ -90,4 +90,5 @@ CMF.XSRC STR {CHIP.NAME}.xsrc # use .PSF and .EXT? CMF.XFIT STR {CHIP.NAME}.xfit # use .PSF and .EXT? + CMF.DETEFF STR {CHIP.NAME}.deteff PSF.HEAD STR {CHIP.NAME}.hdr Index: branches/eam_branches/20090820/ippconfig/mosaic2/format.config =================================================================== --- branches/eam_branches/20090820/ippconfig/mosaic2/format.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/mosaic2/format.config (revision 25766) @@ -90,5 +90,5 @@ CELL.TIMESYS STR UTC # Header says "UTC approximate" CELL.READDIR S32 1 # Cell is read in x direction - CELL.BAD S32 0 + CELL.BAD S32 -100 CELL.YPARITY S32 1 CELL.Y0 S32 0 Index: branches/eam_branches/20090820/ippconfig/mosaic2/psphot.config =================================================================== --- branches/eam_branches/20090820/ippconfig/mosaic2/psphot.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/mosaic2/psphot.config (revision 25766) @@ -1,44 +1,32 @@ -# turn these on to see specific outputs -SAVE.RESID BOOL TRUE +IMSTATS_NPIX S32 10000 # number of pixels to use for sky estimate boxes: -IMSTATS_NPIX S32 1000 # number of pixels to use for sky estimate boxes: +# robust vs fitted?? gpc1 uses fitted... robust median gives better background stdev --> test with flat-field photometry +SKY_STAT STR ROBUST_MEDIAN # statistic used to measure background -PSF_SN_LIM F32 20 # minimum S/N for stars used for PSF model -PSF_MAX_NSTARS S32 300 # limit number of stars used for PSF model +PSF_MODEL STR PS_MODEL_QGAUSS +PSF_MAX_NSTARS S32 300 # limit number of stars used for PSF model -# PSF model parameters : choose the PSF model -# list as many PSF_MODEL options as desired -# if you want to list multiple entries, uncomment 'MULTI' here -# PSF_MODEL MULTI -PSF_MODEL STR PS_MODEL_QGAUSS -# PSF_MODEL STR PS_MODEL_GAUSS -# PSF_MODEL STR PS_MODEL_PGAUSS -# PSF_MODEL STR PS_MODEL_TGAUSS ## not well tested, not very successful +# Mosaic 2 has substantial IQ variations in the y-direction +PSF_CLUMP_NX S32 1 # subdivide image in to NX x NY regions for +PSF_CLUMP_NY S32 3 # selecting PSF stars -MOMENTS_SN_MIN F32 10.0 # min S/N to measure moments -EXT_MIN_SN F32 20.0 # fit galaxies above this S/N limit -FULL_FIT_SN_LIM F32 20.0 -AP_MIN_SN F32 25.0 +# Mosaic 2 has substantial IQ variations in the y-direction +PSF.TREND.NX S32 1 +PSF.TREND.NY S32 3 -USE_FOOTPRINTS BOOL TRUE # use new pmFootprint peak packaging +PEAKS_NSIGMA_LIMIT F32 15.0 # peak significance threshold +FOOTPRINT_NSIGMA_LIMIT F32 14.0 # threshold for bright pmFootprint detection -## gene's overrides for testing: -PEAKS_NSIGMA_LIMIT_2 F32 10.0 # peak significance threshold +PSF_SN_LIM F32 20.0 # minimum S/N for stars used for PSF model +FULL_FIT_SN_LIM F32 10.0 +EXT_MIN_SN F32 10.0 # fit galaxies above this S/N limit -EXT_MODEL STR PS_MODEL_QGAUSS +EXT_MODEL STR PS_MODEL_QGAUSS -TEST_FIT BOOL FALSE -TEST_FIT_X F32 781 -TEST_FIT_Y F32 830 +EXTENDED_SOURCE_SN_LIM F32 3.0 +#EXTENDED_SOURCE_ANALYSIS BOOL TRUE # perform any of the aperture-like measurements? +#EXTENDED_SOURCE_PETROSIAN BOOL TRUE -TEST_FIT_MODE STR CONV -#TEST_FIT_MODE STR PSF -#TEST_FIT_MODE STR EXT - -#TEST_FIT_MODEL STR PS_MODEL_SERSIC -TEST_FIT_MODEL STR PS_MODEL_QGAUSS -TEST_MOMENTS_RADIUS F32 9.0 -OUTPUT.FORMAT STR PS1_DEV_1 - -PSF_MOMENTS_RADIUS F32 7 # calculate initial source moments with this radius +# 3 to match PSF and Moments 2D variations +APTREND.ORDER.MAX S32 3 Index: branches/eam_branches/20090820/ippconfig/recipes/filerules-mef.mdc =================================================================== --- branches/eam_branches/20090820/ippconfig/recipes/filerules-mef.mdc (revision 25206) +++ branches/eam_branches/20090820/ippconfig/recipes/filerules-mef.mdc (revision 25766) @@ -149,6 +149,6 @@ PPIMAGE.OUT.WT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.wt.fits VARIANCE NONE CHIP TRUE SPLIT PPIMAGE.OUTPUT.DETMASK OUTPUT {OUTPUT}.fits IMAGE MASK CHIP TRUE MEF -PPIMAGE.OUTPUT.DETREND OUTPUT {OUTPUT}.fits IMAGE COMP_DET CHIP TRUE MEF -PPIMAGE.OUTPUT.RESID OUTPUT {OUTPUT}.b0.fits IMAGE COMP_SUB CHIP TRUE MEF +PPIMAGE.OUTPUT.DETREND OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE MEF +PPIMAGE.OUTPUT.RESID OUTPUT {OUTPUT}.b0.fits IMAGE NONE CHIP TRUE MEF PPIMAGE.CONFIG OUTPUT {OUTPUT}.{CHIP.NAME}.ppImage.mdc TEXT NONE CHIP TRUE NONE Index: branches/eam_branches/20090820/ippconfig/recipes/filerules-split.mdc =================================================================== --- branches/eam_branches/20090820/ippconfig/recipes/filerules-split.mdc (revision 25206) +++ branches/eam_branches/20090820/ippconfig/recipes/filerules-split.mdc (revision 25766) @@ -119,6 +119,6 @@ PPIMAGE.OUTPUT.VARIANCE OUTPUT {OUTPUT}.{CHIP.NAME}.wt.fits VARIANCE COMP_WT CHIP TRUE NONE PPIMAGE.OUTPUT.DETMASK OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE DET_MASK CHIP TRUE NONE -PPIMAGE.OUTPUT.DETREND OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE COMP_DET CHIP TRUE NONE -PPIMAGE.OUTPUT.RESID OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE COMP_SUB CHIP TRUE NONE +PPIMAGE.OUTPUT.DETREND OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE NONE +PPIMAGE.OUTPUT.RESID OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE NONE PPIMAGE.CONFIG OUTPUT {OUTPUT}.{CHIP.NAME}.ppImage.mdc TEXT NONE CHIP TRUE NONE @@ -127,5 +127,5 @@ PPIMAGE.CHIP.MASK OUTPUT {OUTPUT}.{CHIP.NAME}.ch.mk.fits MASK COMP_MASK CHIP TRUE NONE PPIMAGE.CHIP.VARIANCE OUTPUT {OUTPUT}.{CHIP.NAME}.ch.wt.fits VARIANCE COMP_WT CHIP TRUE NONE -PPIMAGE.CHIP.RESID OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits IMAGE COMP_SUB CHIP TRUE NONE +PPIMAGE.CHIP.RESID OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits IMAGE NONE CHIP TRUE NONE PPIMAGE.OUTPUT.FPA1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE @@ -146,7 +146,7 @@ PPMERGE.OUTPUT.SHUTTER OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE NONE PPMERGE.OUTPUT.FLAT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE NONE -PPMERGE.OUTPUT.FRINGE OUTPUT {OUTPUT}.{CHIP.NAME}.fits FRINGE NONE CHIP TRUE NONE -PPMERGE.OUTPUT.SIGMA OUTPUT {OUTPUT}.{CHIP.NAME}.sigma.fits IMAGE NONE CHIP TRUE NONE -PPMERGE.OUTPUT.COUNT OUTPUT {OUTPUT}.{CHIP.NAME}.count.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.FRINGE OUTPUT {OUTPUT}.{CHIP.NAME}.fits FRINGE NONE CELL TRUE NONE +PPMERGE.OUTPUT.SIGMA OUTPUT {OUTPUT}.{CHIP.NAME}.sigma.fits IMAGE NONE CELL TRUE NONE +PPMERGE.OUTPUT.COUNT OUTPUT {OUTPUT}.{CHIP.NAME}.count.fits IMAGE NONE CELL TRUE NONE DVOCORR.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fc.fits IMAGE NONE CHIP TRUE NONE Index: branches/eam_branches/20090820/ippconfig/recipes/masks.16bit.config =================================================================== --- branches/eam_branches/20090820/ippconfig/recipes/masks.16bit.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/recipes/masks.16bit.config (revision 25766) @@ -20,4 +20,5 @@ # Mask values which represent non-astronomical structures +BURNTOOL U16 0x0080 # Pixel may contain uncorrected streak. CR U16 0x0100 # Pixel contains a cosmic ray SPIKE U16 0x0200 # Pixel contains a diffraction spike Index: branches/eam_branches/20090820/ippconfig/recipes/ppImage.config =================================================================== --- branches/eam_branches/20090820/ippconfig/recipes/ppImage.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/recipes/ppImage.config (revision 25766) @@ -16,4 +16,5 @@ MASK.SATURATED BOOL TRUE # Mask the saturated pixels MASK.LOW BOOL TRUE # Mask pixels below valid range +MASK.BURNTOOL BOOL FALSE # Mask potential burntool trails VARIANCE.BUILD BOOL FALSE # Build internal variance image PATTERN BOOL FALSE # Fit and remove pattern noise? @@ -25,4 +26,5 @@ CHECK.CTE BOOL FALSE # measure CTE errors? USE.DEBURNED.IMAGE BOOL FALSE # use burntool-repaired image? +USE.BEST.BURNTOOL BOOL TRUE # require the best burntooled data TILTYSTREAK.APPLY BOOL FALSE # apply the 'tiltystreak' tool @@ -55,4 +57,7 @@ SCAN.ROWS S32 100 +# Which regions identified by burntool to mask. 0x01 : only unfit trails, 0x02 "up" trails, 0x04 "down" trails +BURNTOOL.TRAILS U16 0x01 + # Non-linearity correction NONLIN.SOURCE STR CHIP.NAME # How to determine the source @@ -145,4 +150,5 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails FRINGE BOOL TRUE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -168,4 +174,5 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails FRINGE BOOL TRUE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -191,4 +198,5 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails FRINGE BOOL TRUE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -557,4 +565,5 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails FRINGE BOOL TRUE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -1717,4 +1726,5 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails PATTERN BOOL TRUE # Subtract pattern noise? FRINGE BOOL TRUE # Fringe subtraction Index: branches/eam_branches/20090820/ippconfig/recipes/ppSim.config =================================================================== --- branches/eam_branches/20090820/ippconfig/recipes/ppSim.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/recipes/ppSim.config (revision 25766) @@ -50,7 +50,7 @@ STARS.FAKE BOOL TRUE # Add fake stars, randomly distributed? -STARS.LUM F32 0.6 # Stellar luminosity function slope (magnitude slope) -STARS.MAG F32 15.5 # Brightest magnitude for fake stars -STARS.DENSITY F32 50.0 # Stellar density (per square degree) at the brightest magnitude +STARS.LUM F32 0.35 # Stellar luminosity function slope (magnitude slope) +STARS.MAG F32 15.5 # Brightest magnitude for fake stars (choose based on real catalog limit) +STARS.DENSITY F32 1.0 # Stellar density (per square degree) at the brightest magnitude STARS.SIGMA.LIM F32 5.0 # significance of faintest sources @@ -185,4 +185,10 @@ END +STACKTEST.MAKE METADATA +END + +STACKTEST.RUN METADATA +END + # Overrides for creating large holes in the image LARGEHOLES METADATA Index: branches/eam_branches/20090820/ippconfig/recipes/ppStack.config =================================================================== --- branches/eam_branches/20090820/ippconfig/recipes/ppStack.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/recipes/ppStack.config (revision 25766) @@ -3,5 +3,5 @@ CONVOLVE BOOL TRUE # Convolve images when stacking? ITER S32 1 # Number of rejection iterations -COMBINE.REJ F32 2.0 # Rejection threshold in combination (sigma) +COMBINE.REJ F32 2.5 # Rejection threshold in combination (sigma) COMBINE.SYS F32 0.03 # Relative systematic error in combination COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean @@ -11,7 +11,7 @@ POOR.FRACTION F32 0.01 # Maximum fraction of bad variance for poor pixels THRESHOLD.MASK F32 0.5 # Threshold for mask deconvolution (0..1) -DECONV.LIMIT F32 5.0 # Deconvolution fraction for rejecting entire image +DECONV.LIMIT F32 1.3 # Deconvolution fraction for rejecting entire image IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image -MATCH.REJ F32 5.0 # Rejection threshold for chi^2 values from matching +MATCH.REJ F32 3.0 # Rejection threshold for chi^2 values from matching ROWS S32 64 # Number of rows to read at once VARIANCE BOOL TRUE # Use variance in rejection? @@ -19,11 +19,11 @@ BIN1 S32 4 # Binning factor for first level BIN2 S32 4 # Binning factor for second level +PHOTOMETRY BOOL TRUE # Do basic photometry? SKIP.BG.SUB BOOL FALSE # Bypass background subtraction? -RENORM BOOL FALSE # Renormalise variance maps? -RENORM.NUM S32 10000 # Number of samples for renormalisation -RENORM.MEAN STR ROBUST_MEDIAN # Statistic to use for mean in renormalisation -RENORM.STDEV STR ROBUST_STDEV # Statistic to use for stdev in renormalisation -RENORM.WIDTH F32 2.2 # Gaussian width for normalisation +RENORM BOOL TRUE # Renormalise weight maps? +RENORM.NUM S32 100000 # Number of samples for renormalisation +RENORM.MIN F32 NAN # Minimum value for renormlisation +RENORM.MAX F32 NAN # Maximum value for renormalisation ### The PHOT mode is intended as a quick and dirty stopgap; it should disappear soon. Index: branches/eam_branches/20090820/ippconfig/recipes/ppStats.config =================================================================== --- branches/eam_branches/20090820/ippconfig/recipes/ppStats.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/recipes/ppStats.config (revision 25766) @@ -190,4 +190,6 @@ CONCEPT STR FPA.TELTEMP.EXTRA # Miscellaneous temperatures (C) + CONCEPT STR FPA.BURNTOOL.APPLIED + CONCEPT STR FPA.PON.TIME # time since last power on Index: branches/eam_branches/20090820/ippconfig/recipes/ppStatsFromMetadata.config =================================================================== --- branches/eam_branches/20090820/ippconfig/recipes/ppStatsFromMetadata.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/recipes/ppStatsFromMetadata.config (revision 25766) @@ -44,4 +44,5 @@ ENTRY VAL FPA.TELTEMP.EXTRA F32 CONSTANT -teltemp_extra # Miscellaneous temperatures (C) ENTRY VAL FPA.PON.TIME F32 CONSTANT -pon_time # time since last power on +# ENTRY VAL FPA.BURNTOOL.APPLIED S32 CONSTANT -burntool_state # ENTRY VAL CHIP.TEMP F32 SAMPLE_MEAN -ccd_temp # CCD temperature ENTRY VAL CELL.EXPOSURE F32 SAMPLE_MEAN -exp_time # Exposure time Index: branches/eam_branches/20090820/ippconfig/recipes/ppSub.config =================================================================== --- branches/eam_branches/20090820/ippconfig/recipes/ppSub.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/recipes/ppSub.config (revision 25766) @@ -21,5 +21,5 @@ POOR.FRACTION F32 0.10 # Maximum fraction of bad weight for poor pixels BADFRAC F32 0.95 # Maximum fraction of bad pixels -@ISIS.WIDTHS F32 1 3 5 7 # Gaussian FWHMs for ISIS kernels +@ISIS.WIDTHS F32 3 5 7 10 # Gaussian FWHMs for ISIS kernels @ISIS.ORDERS S32 2 2 2 2 # Polynomial orders for ISIS kernels SPAM.BINNING S32 2 # Binning in outer region for SPAM kernels @@ -37,16 +37,10 @@ OPTIMUM.ORDER S32 2 # Maximum polynomial order for optimum search -RENORM BOOL FALSE # Renormalise weight maps? -RENORM.NUM S32 10000 # Number of samples for renormalisation -RENORM.MEAN STR ROBUST_MEDIAN # Statistic to use for mean in renormalisation -RENORM.STDEV STR ROBUST_STDEV # Statistic to use for stdev in renormalisation -RENORM.WIDTH F32 2.2 # Gaussian width for normalisation +RENORM BOOL TRUE # Renormalise weight maps? +RENORM.NUM S32 100000 # Number of samples for renormalisation +RENORM.MIN F32 0.6 # Minimum value for renormlisation +RENORM.MAX F32 1.5 # Maximum value for renormalisation INTERPOLATION STR LANCZOS3 # Interpolation mode for bad pixels - -VARIANCE.RESCALE BOOL FALSE # empirically rescale the variance? -VARIANCE.RESCALE.NSAMPLE S32 100000 # number of pixels to select from image for measurement -VARIANCE.RESCALE.MIN.VALID F32 0.66 # do not rescale the variance if the measured sigma less than this value (where 1.0 is expected) -VARIANCE.RESCALE.MAX.VALID F32 1.50 # do not rescale the variance if the measured sigma more than this value (where 1.0 is expected) DUAL BOOL FALSE # Dual convolution? Index: branches/eam_branches/20090820/ippconfig/recipes/psastro.config =================================================================== --- branches/eam_branches/20090820/ippconfig/recipes/psastro.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/recipes/psastro.config (revision 25766) @@ -3,4 +3,7 @@ PSASTRO.ONLY.REFSTARS BOOL FALSE # skip all but refstar matches PSASTRO.SAVE.REFMATCH BOOL FALSE # save refstar matches as table in output smf file + +# select which WCS style to use on output images. +PSASTRO.WCS.USECDKEYS BOOL FALSE # perform single-chip astrometry? Index: branches/eam_branches/20090820/ippconfig/recipes/psphot.config =================================================================== --- branches/eam_branches/20090820/ippconfig/recipes/psphot.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/recipes/psphot.config (revision 25766) @@ -2,12 +2,12 @@ # these options turn on different inputs and/or outputs SAVE.OUTPUT BOOL TRUE -SAVE.BACKMDL BOOL FALSE +SAVE.BACKMDL BOOL TRUE SAVE.BACKMDL.STDEV BOOL FALSE SAVE.BACKGND BOOL FALSE SAVE.BACKSUB BOOL FALSE SAVE.RESID BOOL FALSE +SAVE.PLOTS BOOL FALSE SAVE.PSF BOOL TRUE LOAD.PSF BOOL FALSE -SAVE.PLOTS BOOL FALSE # format for output CMF files @@ -15,7 +15,5 @@ # the zero point is used to set a basic scale for DVO -# XXX it may not currently be read : double check this (EAM) -ZERO_POINT F32 25.000 # zero point used by DVO -ZERO_PT F32 25.000 # zero point used by DVO +ZERO_PT F32 25.000 # zero point used by DVO XXX deprecate this # these parameter govern how the background is measured @@ -23,4 +21,8 @@ BACKGROUND.YBIN S32 128 # size of background superpixels IMSTATS_NPIX S32 10000 # number of pixels to use for sky estimate boxes: + +# soften the errors for bright pixels: +SOFTEN.VARIANCE BOOL FALSE +SOFTEN.VARIANCE.FRACTION F32 0.0 SKY_BIAS F32 0.0 # offset applied to measured sky (FOR TESTING) @@ -43,26 +45,38 @@ PEAKS_SMOOTH_SIGMA F32 1.0 # smoothing kernel sigma in pixels PEAKS_SMOOTH_NSIGMA F32 2.0 # smoothing kernel width in sigmas -PEAKS_NSIGMA_LIMIT F32 25.0 # peak significance threshold +PEAKS_NSIGMA_LIMIT F32 20.0 # peak significance threshold PEAKS_NSIGMA_LIMIT_2 F32 5.0 # peak significance threshold -PEAKS_NMAX S32 0 # on first pass, only keep NMAX peaks (0 == all) -PEAKS_MIN_GAUSS F32 0.5 # Minimum valid fraction of kernel +PEAKS_NMAX S32 5000 # on first pass, only keep NMAX peaks (0 == all) +PEAKS_MIN_GAUSS F32 0.5 # On quick convolution, mask pixels for which the + # input pixels contribute less than this fraction of the flux +# parameters which adjust the footprint analysis +USE_FOOTPRINTS BOOL TRUE # use new pmFootprint peak packaging +FOOTPRINT_NPIXMIN S32 5 # Minimum size of a pmFootprint +FOOTPRINT_GROW_RADIUS S32 3 # How much to grow bright footprints +FOOTPRINT_GROW_RADIUS_2 S32 5 # How much to grow faint footprints +FOOTPRINT_CULL_NSIGMA_DELTA F32 4 # Cull peaks that aren't nsigma above coll to neighbour +FOOTPRINT_CULL_NSIGMA_MIN F32 1 # Minimum height of colls in units of skyStdev +FOOTPRINT_CULL_NSIGMA_PAD F32 0.01 # Fractional Padding for stdev + +# parameter for the simple deblending +DEBLEND_PEAK_FRACTION F32 0.1 +DEBLEND_SKY_NSIGMA F32 10.0 # parameters to control the selection of the peak in the Sx,Sy plane MOMENTS_SCALE F32 0.25 -MOMENTS_SN_MIN F32 10.0 # min S/N to measure moments +MOMENTS_SN_MIN F32 0.0 # min S/N to measure moments (default: measure all objects) MOMENTS_SX_MAX F32 50.0 MOMENTS_SY_MAX F32 50.0 MOMENTS_AR_MAX F32 1.5 # maximum axial ratio: 1 / AR < (sx / sy) < AR -MOMENTS_MIN_PIXEL_SN F32 0.0 # XXX caution on this: too high and we will excessively clip -MOMENTS_GAUSS_SIGMA F32 4.0 # XXX TEST THIS: sigma in pixels (should be >> seeing equiv) +MOMENTS_GAUSS_SIGMA F32 4.0 # XXX this is now autoscaled # basic object statistics SKY_INNER_RADIUS F32 15 # square annulus for local sky measurement SKY_OUTER_RADIUS F32 25 # square annulus for local sky measurement -PSF_MOMENTS_RADIUS F32 3 # calculate initial source moments with this radius -PSF_SN_LIM F32 50 # minimum S/N for stars used for PSF model -PSF_MAX_NSTARS S32 200 # limit number of stars used for PSF model +PSF_MOMENTS_RADIUS F32 3 # XXX this is now autoscaled +PSF_SN_LIM F32 20 # minimum S/N for stars used for PSF model +PSF_MAX_NSTARS S32 500 # limit number of stars used for PSF model PSF_CLUMP_NSIGMA F32 1.5 # region of Sx,Sy plane to use for selecting PSF stars -PSF_CLUMP_GRID_SCALE F32 0.5 # size of Sx,Sy image pixel +PSF_CLUMP_GRID_SCALE F32 0.2 # size of Sx,Sy image pixel PSF_CLUMP_NX S32 1 # subdivide image in to NX x NY regions for @@ -80,6 +94,4 @@ PSF_MODEL STR PS_MODEL_GAUSS # PSF_MODEL STR PS_MODEL_PGAUSS -# PSF_MODEL STR PS_MODEL_QGAUSS -# PSF_MODEL STR PS_MODEL_TGAUSS # not well tested, not very successful # PSF.TREND.MASK must be a 2D polynomial @@ -98,13 +110,16 @@ PSF.FAKE.ALLOW BOOL FALSE # Allow use of fake PSFs when having trouble? -PSF_FIT_RADIUS F32 15.0 # fitting radius for test PSF model +PSF_FIT_RADIUS F32 15.0 # XXX this is now autoscaled PSF_REF_RADIUS F32 25.0 # aperture magnitudes are scaled via - # curve-of-growth to this radius + # curve-of-growth to this radius + +PSF_FIT_RADIUS_SCALE F32 7.0 # fitting radius for test PSF model +PSF_APERTURE_SCALE F32 4.5 # fitting radius for test PSF model + # PSF-like source model parameters PSF_FIT_NSIGMA F32 1.0 # significance for pixel included in fit PSF_FIT_PADDING F32 2.0 # extra annulus to use for fit -PSF_SHAPE_NSIGMA F32 3.0 # max significance for shape variation PSF_MIN_SN F32 2.0 # reject objects below this significance -PSF_MAX_CHI F32 50.0 # reject objects worse that this +PSF_MAX_CHI F32 50.0 # reject objects worse than this FULL_FIT_SN_LIM F32 50.0 @@ -117,12 +132,13 @@ PSF.RESIDUALS.STATISTIC STR ROBUST_MEDIAN # statistic to use for generating the residual PSF.RESIDUALS.SPATIAL_ORDER S32 0 # fit spatial variations of the residuals at this order (0,1) -PSF.RESIDUALS.PIX.SN F32 5.0 # keep this pixel if residual is more significant than this +PSF.RESIDUALS.PIX.SN F32 0.0 # keep this pixel if residual is more significant than this +PSF.RESIDUALS.RADIUS F32 8.0 # keep this pixel if residual is more significant than this # this model is used for approximate subtraction in the main object analysis step EXT_MODEL STR PS_MODEL_QGAUSS -EXT_MIN_SN F32 50.0 # fit galaxies above this S/N limit EXT_FIT_NSIGMA F32 1 # significance for pixel included in fit EXT_FIT_PADDING F32 5 # extra annulus to use for fit EXT_MOMENTS_RADIUS F32 31 +EXT_FIT_MAX_RADIUS F32 128 # non-sensical to fit objects larger than the background mesh EXT_FIT_ITER S32 20 # Maximum number of fitting iterations for EXT @@ -172,12 +188,8 @@ PETROSIAN_FLUX_RATIO F32 0.1 # we measure out to where flux = R0 * FLUX_RATIO -FITMODE STR BLEND -DEBLEND_PEAK_FRACTION F32 0.1 -DEBLEND_SKY_NSIGMA F32 10.0 - # APTREND STR NONE, CONSTANT, SKYBIAS, SKYSAT, XY_LIN, SKY_XY_LIN, SKYSAT_XY_LIN, ALL -MEASURE.APTREND BOOL FALSE ### Gene says there's bugs in this part +MEASURE.APTREND BOOL TRUE APTREND STR CONSTANT -AP_MIN_SN F32 25.0 +AP_MIN_SN F32 0.0 # measure apMags for all sources by default APTREND.NSTAR.MIN S32 5 APTREND.ORDER.MAX S32 5 @@ -196,5 +208,5 @@ IGNORE_GROWTH BOOL FALSE -CONSTANT_PHOTOMETRIC_WEIGHTS BOOL TRUE # Should the photometric code [currently only ensemblePSF] refuse to weight each pixel by it's significance? +CONSTANT_PHOTOMETRIC_WEIGHTS BOOL FALSE INTERPOLATE_AP BOOL TRUE @@ -207,14 +219,4 @@ NOISE.FACTOR F32 5.0 NOISE.SIZE F32 2.0 - -USE_FOOTPRINTS BOOL F # use new pmFootprint peak packaging -FOOTPRINT_NPIXMIN S32 5 # Minimum size of a pmFootprint -FOOTPRINT_NSIGMA_LIMIT F32 20 # threshold for bright pmFootprint detection -FOOTPRINT_NSIGMA_LIMIT_2 F32 4 # threshold for faint pmFootprint detection -FOOTPRINT_GROW_RADIUS S32 3 # How much to grow bright footprints -FOOTPRINT_GROW_RADIUS_2 S32 5 # How much to grow faint footprints -FOOTPRINT_CULL_NSIGMA_DELTA F32 4 # Cull peaks that aren't nsigma above coll to neighbour -FOOTPRINT_CULL_NSIGMA_MIN F32 1 # Minimum height of colls in units of skyStdev -FOOTPRINT_CULL_NSIGMA_PAD F32 0.01 # Fractional Padding for stdev # alternate PSPHOT-type recipes @@ -261,8 +263,12 @@ PSPHOT.CR.NSIGMA.LIMIT F32 3.0 # sources with crNsigma greater that this get tagged as likely cosmic rays -PSPHOT.EXT.NSIGMA.LIMIT F32 4.0 # sources with extNsigma greater that this get tagged as likely extended sources +PSPHOT.EXT.NSIGMA.LIMIT F32 3.0 # sources with extNsigma greater that this get tagged as likely extended sources +PSPHOT.EXT.NSIGMA.MOMENTS F32 2.0 # sources with extNsigma greater that this get tagged as likely extended sources PSPHOT.CR.GROW S32 1 # Number of pixels to grow CR mask PSPHOT.CR.NSIGMA.SOFTEN F32 0.0025 # Softening parameter for weights +# Detection efficiency +EFF.NUM S32 500 # Number of fake sources per bin +@EFF.MAG F32 -2.0 -1.0 -0.5 -0.25 -0.1 -0.05 0.0 0.05 0.1 0.25 0.5 1.0 2.0 # Magnitude of fake sources relative to limit # Recipe overrides for CHIP Index: branches/eam_branches/20090820/ippconfig/recipes/pswarp.config =================================================================== --- branches/eam_branches/20090820/ippconfig/recipes/pswarp.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/recipes/pswarp.config (revision 25766) @@ -10,6 +10,12 @@ ACCEPT.FRAC F32 0.1 # Minimum fraction of good pixels to accept result INTERPOLATION.NUM S32 1000 # Number of interpolation kernels to pre-calculate +PSF BOOL TRUE # Measure PSF for warped image? # Default recipe for warping WARP METADATA END + +#PR recipe for faster processing +PR METADATA + PSF BOOL FALSE +END Index: branches/eam_branches/20090820/ippconfig/recipes/reductionClasses.mdc =================================================================== --- branches/eam_branches/20090820/ippconfig/recipes/reductionClasses.mdc (revision 25206) +++ branches/eam_branches/20090820/ippconfig/recipes/reductionClasses.mdc (revision 25766) @@ -307,5 +307,5 @@ CHIP_PPIMAGE STR PR CHIP_PSPHOT STR PR - WARP_PSWARP STR WARP + WARP_PSWARP STR PR STACK_PPSTACK STR PR STACK_PPSUB STR PR @@ -324,5 +324,5 @@ CHIP_PPIMAGE STR PR_DARKTEST_NOSUB CHIP_PSPHOT STR PR - WARP_PSWARP STR WARP + WARP_PSWARP STR PR STACK_PPSTACK STR PR STACK_PPSUB STR PR Index: branches/eam_branches/20090820/ippconfig/sdss/psphot.config =================================================================== --- branches/eam_branches/20090820/ippconfig/sdss/psphot.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/sdss/psphot.config (revision 25766) @@ -1,61 +1,8 @@ - -# turn these on to see specific outputs -SAVE.BACKMDL BOOL TRUE -#SAVE.BACKGND BOOL TRUE -#SAVE.BACKSUB BOOL TRUE -#SAVE.RESID BOOL TRUE -SAVE.PSF BOOL TRUE -SAVE.PLOTS BOOL TRUE - -# image statistics parameters -IMSTATS_NPIX S32 3000 # number of pixels to use for sky estimate boxes: - -PSF_SN_LIM F32 10 # minimum S/N for stars used for PSF model -PSF_MAX_NSTARS S32 300 # limit number of stars used for PSF model PSF_MODEL STR PS_MODEL_QGAUSS -MOMENTS_SN_MIN F32 10.0 -#EXT_MIN_SN F32 50.0 # fit galaxies above this S/N limit -#FULL_FIT_SN_LIM F32 50.0 -#AP_MIN_SN F32 20.0 +# check: PSF_CLUMP_NSIGMA F32 2.5 # region of Sx,Sy plane to use for selecting PSF stars -# PSFTREND must be a 2D polynomial -# the specified values are ignored but define the active components of the polynomial -PSF.TREND.MASK METADATA - NORDER_X S32 3 # number of x orders - NORDER_Y S32 3 # number of y orders - VAL_X00_Y00 F64 1 # polynomial coefficient - - VAL_X01_Y00 F64 1 # polynomial coefficient - VAL_X00_Y01 F64 1 # polynomial coefficient - - VAL_X02_Y00 F64 1 # polynomial coefficient - VAL_X01_Y01 F64 1 # polynomial coefficient - VAL_X00_Y02 F64 1 # polynomial coefficient - - VAL_X03_Y00 F64 1 # polynomial coefficient - VAL_X02_Y01 F64 1 # polynomial coefficient - VAL_X01_Y02 F64 1 # polynomial coefficient - VAL_X00_Y03 F64 1 # polynomial coefficient - NELEMENTS S32 10 # number of unmasked components -END # folder for 4D polynomial - +# is this needed to mask bad regions? XMIN F32 15 - -PSF.RESIDUALS BOOL true -PSF.RESIDUALS.SPATIAL_ORDER S32 1 - -# BREAK_POINT STR ENSEMBLE -OUTPUT.FORMAT STR PS1_DEV_0 - -PSPHOT.SUMMIT METADATA - PEAKS_SMOOTH_SIGMA F32 0.8 # peak significance threshold - PEAKS_NSIGMA_LIMIT F32 50.0 # peak significance threshold - PEAKS_NMAX S32 1000 - SAVE.RESID BOOL FALSE - PSF_SN_LIM F32 25 # minimum S/N for stars used for PSF model - MOMENTS_SN_MIN F32 25.0 - PSF_MODEL STR PS_MODEL_PGAUSS -END Index: branches/eam_branches/20090820/ippconfig/sdssmosaic/psphot.config =================================================================== --- branches/eam_branches/20090820/ippconfig/sdssmosaic/psphot.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/sdssmosaic/psphot.config (revision 25766) @@ -1,42 +1,15 @@ - -# turn these on to see specific outputs -SAVE.OUTPUT BOOL TRUE -SAVE.BACKMDL BOOL TRUE -#SAVE.RESID BOOL TRUE -#SAVE.BACKGND BOOL TRUE -SAVE.BACKSUB BOOL TRUE -SAVE.PSF BOOL TRUE -#LOAD.PSF BOOL FALSE -#SAVE.PLOTS BOOL TRUE - -# for testing: -# BREAK_POINT STR BACKMDL - BACKGROUND.XBIN S32 256 # size of background superpixels BACKGROUND.YBIN S32 256 # size of background superpixels -# image background parameters -IMSTATS_NPIX S32 10000 # number of pixels to use for sky estimate boxes: -SKY_STAT STR FITTED_MEAN_V4 # statistic used to measure background -SKY_CLIP_SIGMA F32 2.0 # statistic used to measure background - -PEAKS_SMOOTH_SIGMA F32 2.5 # smoothing kernel sigma in pixels -PEAKS_NMAX S32 5000 # on first pass, only keep NMAX peaks (0 == all) -PEAKS_NSIGMA_LIMIT F32 25.0 # peak significance threshold -PEAKS_NSIGMA_LIMIT_2 F32 5.0 # peak significance threshold - -PSF_SN_LIM F32 25 # minimum S/N for stars used for PSF model -PSF_MAX_NSTARS S32 300 # limit number of stars used for PSF model +# what is the 2D variation of the SDSS PSF? PSF_CLUMP_NX S32 3 # subdivide image in to NX x NY regions for PSF_CLUMP_NY S32 3 # selecting PSF stars -PSF_MOMENTS_RADIUS F32 12 # calculate initial source moments with this radius + +# test this: PSF_CLUMP_GRID_SCALE F32 2.5 # size of Sx,Sy image pixel - -PSF.RESIDUALS BOOL TRUE # generate the residuals? # PSF model parameters : choose the PSF model desired PSF_MODEL STR PS_MODEL_PS1_V1 -PSPHOT.CR.NSIGMA.SOFTEN F32 0.0025 # Softening parameter for weights # Use same default setting as in 2.6.1 (MAP is now global default) @@ -45,32 +18,8 @@ PSF.TREND.NY S32 0 -EXT_MIN_SN F32 50.0 # fit galaxies above this S/N limit -FULL_FIT_SN_LIM F32 25.0 -MOMENTS_SN_MIN F32 5.0 -AP_MIN_SN F32 5.0 - -OUTPUT.FORMAT STR PS1_V1 - -PSF_SHAPE_NSIGMA F32 3.0 # max significance for shape variation -PSF_MAX_CHI F32 50.0 # reject objects worse that this - -MEASURE.APTREND BOOL TRUE ### XXX for now, skip this (too many errors) -APTREND.ORDER.MAX S32 3 - -# BREAK_POINT STR PASS1 - +# match the SDSS radius: PSF_REF_RADIUS F32 18.7 # aperture magnitudes are scaled via # curve-of-growth to this radius - -DIAGNOSTIC.PLOTS METADATA - IMAGE.BACKGROUND.CELL.HISTOGRAM BOOL FALSE - IMAGE.BACKGROUND.CELL.HISTOGRAM.X S32 32 - IMAGE.BACKGROUND.CELL.HISTOGRAM.Y S32 15 -END - -USE_FOOTPRINTS BOOL TRUE # use new pmFootprint peak packaging - -# Default is noap, nopsf; following recipes turn on one or both of these - +# for tests: Default is noap, nopsf; following recipes turn on one or both of these PSPHOT_NOAP_PSF METADATA PSF.TREND.MODE STR MAP @@ -96,4 +45,5 @@ END +# put these values in the master? DIFF METADATA BACKGROUND.XBIN S32 32 # size of background superpixels Index: branches/eam_branches/20090820/ippconfig/simmosaic/psphot.config =================================================================== --- branches/eam_branches/20090820/ippconfig/simmosaic/psphot.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/simmosaic/psphot.config (revision 25766) @@ -1,40 +1,3 @@ -SAVE.BACKMDL BOOL TRUE -SAVE.BACKGND BOOL TRUE -SAVE.BACKSUB BOOL TRUE -SAVE.PLOTS BOOL FALSE -SAVE.RESID BOOL TRUE -SAVE.PSF BOOL TRUE -LOAD.PSF BOOL FALSE - -IMSTATS_NPIX S32 3000 # number of pixels to use for sky estimate boxes: -SKY_INNER_RADIUS F32 10 # square annulus for local sky measurement - -PSF_MOMENTS_RADIUS F32 7 # calculate initial source moments with this radius - - - -#PSPHOT_TEST METADATA -# SAVE.BACKSUB BOOL FALSE -# SAVE.RESID BOOL TRUE -#END -# -#PEAKS_OUTPUT_FILE STR peaks.dat -#MOMENTS_OUTPUT_FILE STR moments.dat - -#PSF_SN_LIM F32 20.0 # minimum S/N for stars used for PSF model -MOMENTS_SN_MIN F32 10.0 # min S/N to measure moments -FULL_FIT_SN_LIM F32 100.0 -EXT_MIN_SN F32 100.0 # fit galaxies above this S/N limit - -MOMENTS_SX_MAX F32 25.0 -MOMENTS_SY_MAX F32 25.0 - - -PEAKS_NSIGMA_LIMIT F32 25.0 # peak significance threshold -PEAKS_NSIGMA_LIMIT_2 F32 10.0 # peak significance threshold - -BREAK_POINT STR ENSEMBLE # limit total processing for now (for speed) - - -SKY_STAT STR ROBUST_MEDIAN # statistic used to measure background +# ppSim currently uses PS_MODEL_GAUSS +PSF_MODEL STR PS_MODEL_PGAUSS Index: branches/eam_branches/20090820/ippconfig/simple/psphot.config =================================================================== --- branches/eam_branches/20090820/ippconfig/simple/psphot.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/simple/psphot.config (revision 25766) @@ -1,17 +1,2 @@ -SAVE.BACKMDL BOOL FALSE -SAVE.BACKGND BOOL FALSE -SAVE.BACKSUB BOOL FALSE -SAVE.PLOTS BOOL TRUE SAVE.RESID BOOL TRUE -SAVE.PSF BOOL TRUE -LOAD.PSF BOOL FALSE - -IMSTATS_NPIX S32 3000 # number of pixels to use for sky estimate boxes: - -MOMENTS_SN_MIN F32 20.0 # min S/N to measure moments -FULL_FIT_SN_LIM F32 100.0 -EXT_MIN_SN F32 100.0 # fit galaxies above this S/N limit - -PEAKS_NSIGMA_LIMIT F32 10.0 # peak significance threshold -USE_FOOTPRINTS BOOL T # use new pmFootprint peak packaging Index: branches/eam_branches/20090820/ippconfig/simtest/camera.config =================================================================== --- branches/eam_branches/20090820/ippconfig/simtest/camera.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/simtest/camera.config (revision 25766) @@ -64,4 +64,5 @@ CMF.XSRC STR {CHIP.NAME}.xsrc # use .PSF and .EXT? CMF.XFIT STR {CHIP.NAME}.xfit # use .PSF and .EXT? + CMF.DETEFF STR {CHIP.NAME}.deteff PSF.HEAD STR {CHIP.NAME}.hdr Index: branches/eam_branches/20090820/ippconfig/simtest/format.config =================================================================== --- branches/eam_branches/20090820/ippconfig/simtest/format.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/simtest/format.config (revision 25766) @@ -69,6 +69,6 @@ CELL.X0 S32 0 CELL.Y0 S32 0 - CELL.XSIZE S32 2048 - CELL.YSIZE S32 2048 + CELL.XSIZE S32 1024 + CELL.YSIZE S32 1024 CELL.XWINDOW S32 0 CELL.YWINDOW S32 0 Index: branches/eam_branches/20090820/ippconfig/simtest/ppSim.config =================================================================== --- branches/eam_branches/20090820/ippconfig/simtest/ppSim.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/simtest/ppSim.config (revision 25766) @@ -3,4 +3,49 @@ # for a quick test, we are going to skip shutter and dark corrections TESTDATA METADATA + PSF.MODEL STR PS_MODEL_PGAUSS + SHUTTER.TIME F32 0.0 + DARK.RATE F32 0.0 + SCATTER.FRAC F32 0.0 + BIAS BOOL FALSE + FLAT BOOL FALSE + BIAS.LEVEL F32 1000.0 # Bias level (e) + BIAS.RANGE F32 20.0 # Range of bias values (e) + BIAS.ORDER S32 7 # Order for overscan polynomial + + STARS.MAG F32 10.0 # Brightest magnitude for fake stars + STARS.DENSITY F32 100.0 # Stellar density (per square degree) at the brightest magnitude + STARS.SIGMA.LIM F32 5.0 # significance of faintest sources + STARS.LUM F32 0.35 # Stellar luminosity function slope (magnitude slope) +END + +# for a quick test, we are going to skip shutter and dark corrections +STACKTEST.MAKE METADATA + PSF.MODEL STR PS_MODEL_PGAUSS + SHUTTER.TIME F32 0.0 + DARK.RATE F32 0.0 + SCATTER.FRAC F32 0.0 + BIAS BOOL FALSE + FLAT BOOL FALSE + BIAS.LEVEL F32 1000.0 # Bias level (e) + BIAS.RANGE F32 20.0 # Range of bias values (e) + BIAS.ORDER S32 7 # Order for overscan polynomial + + # generate a set of fake stars + STARS.REAL BOOL FALSE # Add stars from a catalogue? + STARS.FAKE BOOL TRUE # Add fake stars, randomly distributed? + MATCH.DENSITY BOOL TRUE # significance of faintest sources + + EXPTIME F32 10.0 + RA F32 5.0 + DEC F32 5.0 + PA F32 0.0 + SEEING F32 1.0 + SCALE F32 0.25 + ZEROPOINT F32 25.0 + SKY.MAGS F32 22.0 +END + +# for a quick test, we are going to skip shutter and dark corrections +STACKTEST.RUN METADATA PSF.MODEL STR PS_MODEL_GAUSS SHUTTER.TIME F32 0.0 @@ -8,7 +53,22 @@ SCATTER.FRAC F32 0.0 BIAS BOOL FALSE + FLAT BOOL FALSE BIAS.LEVEL F32 1000.0 # Bias level (e) BIAS.RANGE F32 20.0 # Range of bias values (e) BIAS.ORDER S32 7 # Order for overscan polynomial + + # generate a set of fake stars + STARS.REAL BOOL TRUE # Add stars from a catalogue? + STARS.FAKE BOOL FALSE # Add fake stars, randomly distributed? + MATCH.DENSITY BOOL TRUE # significance of faintest sources + + EXPTIME F32 10.0 + RA F32 5.0 + DEC F32 5.0 + PA F32 0.0 + SEEING F32 1.0 + SCALE F32 0.25 + ZEROPOINT F32 25.0 + SKY.MAGS F32 22.0 END Index: branches/eam_branches/20090820/ippconfig/simtest/psastro.config =================================================================== --- branches/eam_branches/20090820/ippconfig/simtest/psastro.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/simtest/psastro.config (revision 25766) @@ -3,5 +3,5 @@ PSASTRO.GRID.OFFSET F32 500. -PSASTRO.GRID.SCALE F32 50. +PSASTRO.GRID.SCALE F32 10. PSASTRO.IGNORE STR CRLIMIT,SATURATED,DEFECT,BLEND @@ -9,5 +9,5 @@ PSASTRO.CATDIR STR SYNTH.SIMTEST DVO.GETSTAR.PHOTCODE STR r -PSASTRO.MAX.NSTAR S32 50 # max stars accepted for fitting +PSASTRO.MAX.NSTAR S32 200 # max stars accepted for fitting DVO.GETSTAR.MAX.RHO F32 10000.0 DVO.GETSTAR.MIN.MAG F32 10.0 @@ -34,2 +34,3 @@ PHOTCODE STR z_SYNTH END + Index: branches/eam_branches/20090820/ippconfig/simtest/psphot.config =================================================================== --- branches/eam_branches/20090820/ippconfig/simtest/psphot.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/simtest/psphot.config (revision 25766) @@ -1,36 +1,4 @@ -SAVE.BACKMDL BOOL TRUE -SAVE.BACKGND BOOL TRUE -SAVE.BACKSUB BOOL TRUE -SAVE.PLOTS BOOL TRUE -SAVE.RESID BOOL TRUE -SAVE.PSF BOOL TRUE -LOAD.PSF BOOL FALSE - -IMSTATS_NPIX S32 3000 # number of pixels to use for sky estimate boxes: -SKY_INNER_RADIUS F32 10 # square annulus for local sky measurement - -PSF_MOMENTS_RADIUS F32 7 # calculate initial source moments with this radius - -#PSPHOT_TEST METADATA -# SAVE.BACKSUB BOOL FALSE -# SAVE.RESID BOOL TRUE -#END -# -#PEAKS_OUTPUT_FILE STR peaks.dat -#MOMENTS_OUTPUT_FILE STR moments.dat - -#PSF_SN_LIM F32 20.0 # minimum S/N for stars used for PSF model -MOMENTS_SN_MIN F32 10.0 # min S/N to measure moments -FULL_FIT_SN_LIM F32 100.0 -EXT_MIN_SN F32 100.0 # fit galaxies above this S/N limit - -MOMENTS_SX_MAX F32 25.0 -MOMENTS_SY_MAX F32 25.0 - -PEAKS_NSIGMA_LIMIT F32 25.0 # peak significance threshold -PEAKS_NSIGMA_LIMIT_2 F32 10.0 # peak significance threshold -BREAK_POINT STR NONE # limit total processing for now (for speed) - - -SKY_STAT STR ROBUST_MEDIAN # statistic used to measure background +# same as ppSim.config (if YES, should be 0.0) +# ppSim currently uses PS_MODEL_GAUSS +PSF_MODEL STR PS_MODEL_PGAUSS Index: branches/eam_branches/20090820/ippconfig/tek/psphot.config =================================================================== --- branches/eam_branches/20090820/ippconfig/tek/psphot.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/tek/psphot.config (revision 25766) @@ -1,57 +1,3 @@ - -# turn these on to see specific outputs -#SAVE.BACKMDL BOOL TRUE -#SAVE.BACKGND BOOL TRUE -#SAVE.BACKSUB BOOL TRUE -SAVE.RESID BOOL TRUE -#SAVE.PSF BOOL TRUE - -# image statistics parameters -IMSTATS_NPIX S32 3000 # number of pixels to use for sky estimate boxes: - -PSF_SN_LIM F32 10 # minimum S/N for stars used for PSF model -PSF_MAX_NSTARS S32 300 # limit number of stars used for PSF model PSF_MODEL STR PS_MODEL_QGAUSS -MOMENTS_SN_MIN F32 10.0 -#EXT_MIN_SN F32 50.0 # fit galaxies above this S/N limit -#FULL_FIT_SN_LIM F32 50.0 -#AP_MIN_SN F32 20.0 -PSF_CLUMP_NSIGMA F32 2.5 # region of Sx,Sy plane to use for selecting PSF stars - -# PSFTREND must be a 2D polynomial -# the specified values are ignored but define the active components of the polynomial -PSF.TREND.MASK METADATA - NORDER_X S32 3 # number of x orders - NORDER_Y S32 3 # number of y orders - VAL_X00_Y00 F64 1 # polynomial coefficient - - VAL_X01_Y00 F64 1 # polynomial coefficient - VAL_X00_Y01 F64 1 # polynomial coefficient - - VAL_X02_Y00 F64 1 # polynomial coefficient - VAL_X01_Y01 F64 1 # polynomial coefficient - VAL_X00_Y02 F64 1 # polynomial coefficient - - VAL_X03_Y00 F64 1 # polynomial coefficient - VAL_X02_Y01 F64 1 # polynomial coefficient - VAL_X01_Y02 F64 1 # polynomial coefficient - VAL_X00_Y03 F64 1 # polynomial coefficient - NELEMENTS S32 10 # number of unmasked components -END # folder for 4D polynomial - -XMIN F32 15 -PSF.RESIDUALS BOOL false -BREAK_POINT STR ENSEMBLE -OUTPUT.FORMAT STR PS1_DEV_0 - -PSPHOT.SUMMIT METADATA - PEAKS_SMOOTH_SIGMA F32 0.8 # peak significance threshold - PEAKS_NSIGMA_LIMIT F32 50.0 # peak significance threshold - PEAKS_NMAX S32 1000 - SAVE.RESID BOOL FALSE - PSF_SN_LIM F32 25 # minimum S/N for stars used for PSF model - MOMENTS_SN_MIN F32 25.0 - PSF_MODEL STR PS_MODEL_PGAUSS -END Index: branches/eam_branches/20090820/ippconfig/ucam/psphot.config =================================================================== --- branches/eam_branches/20090820/ippconfig/ucam/psphot.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/ucam/psphot.config (revision 25766) @@ -1,27 +1,2 @@ -# turn these on to see specific outputs -#SAVE.BACKMDL BOOL TRUE -#SAVE.BACKGND BOOL TRUE -#SAVE.BACKSUB BOOL TRUE -SAVE.RESID BOOL TRUE -#SAVE.PSF BOOL TRUE -#LOAD.PSF BOOL FALSE - -IMSTATS_NPIX S32 1000 # number of pixels to use for sky estimate boxes: - -PSF_SN_LIM F32 100 # minimum S/N for stars used for PSF model -PSF_MAX_NSTARS S32 300 # limit number of stars used for PSF model - -# PSF model parameters : choose the PSF model -# list as many PSF_MODEL options as desired -# if you want to list multiple entries, uncomment 'MULTI' here -# PSF_MODEL MULTI PSF_MODEL STR PS_MODEL_QGAUSS -# PSF_MODEL STR PS_MODEL_GAUSS -# PSF_MODEL STR PS_MODEL_PGAUSS -# PSF_MODEL STR PS_MODEL_TGAUSS ## not well tested, not very successful - -MOMENTS_SN_MIN F32 20.0 # min S/N to measure moments -EXT_MIN_SN F32 50.0 # fit galaxies above this S/N limit -FULL_FIT_SN_LIM F32 50.0 -AP_MIN_SN F32 25.0 Index: branches/eam_branches/20090820/ippconfig/uh8k/psphot.config =================================================================== --- branches/eam_branches/20090820/ippconfig/uh8k/psphot.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/uh8k/psphot.config (revision 25766) @@ -1,37 +1,5 @@ - -# turn these on to see specific outputs -SAVE.OUTPUT BOOL TRUE -SAVE.BACKMDL BOOL TRUE -SAVE.PSF BOOL TRUE -SAVE.PLOTS BOOL TRUE - -BACKGROUND.XBIN S32 128 # size of background superpixels -BACKGROUND.YBIN S32 128 # size of background superpixels - -# image background parameters -IMSTATS_NPIX S32 10000 # number of pixels to use for sky estimate boxes: -SKY_STAT STR FITTED_MEAN_V4 # statistic used to measure background -SKY_CLIP_SIGMA F32 2.0 # statistic used to measure background - -PSF_SN_LIM F32 100 # minimum S/N for stars used for PSF model -PSF_MAX_NSTARS S32 300 # limit number of stars used for PSF model # PSF model parameters : choose the PSF model desired PSF_MODEL STR PS_MODEL_QGAUSS - -MOMENTS_SN_MIN F32 30.0 -EXT_MIN_SN F32 50.0 # fit galaxies above this S/N limit -FULL_FIT_SN_LIM F32 50.0 -AP_MIN_SN F32 50.0 - -# OUTPUT.FORMAT STR SMPDATA -OUTPUT.FORMAT STR PS1_DEV_1 - -PSF_SHAPE_NSIGMA F32 3.0 # max significance for shape variation -PSF_MAX_CHI F32 50.0 # reject objects worse that this - -#PSF.TREND.MODE STR MAP -#PSF.TREND.NX S32 1 -#PSF.TREND.NY S32 1 DIAGNOSTIC.PLOTS METADATA @@ -40,4 +8,2 @@ IMAGE.BACKGROUND.CELL.HISTOGRAM.Y S32 12 END - -USE_FOOTPRINTS BOOL TRUE # use new pmFootprint peak packaging Index: branches/eam_branches/20090820/ippconfig/vysos5/psastro.config =================================================================== --- branches/eam_branches/20090820/ippconfig/vysos5/psastro.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/vysos5/psastro.config (revision 25766) @@ -26,4 +26,5 @@ # PSASTRO.GRID.MAX.ANGLE F32 360.0 # PSASTRO.GRID.DEL.ANGLE F32 1.0 + PSASTRO.GRID.MIN.ANGLE F32 -20.0 # start angle (degrees) PSASTRO.GRID.MAX.ANGLE F32 +20.0 @@ -70,4 +71,5 @@ PSASTRO.CATDIR STR SYNTH.BRIGHT DVO.GETSTAR.PHOTCODE STR r +DVO.GETSTAR.MAX.RHO F32 1000.0 PHOTCODE.DATA MULTI @@ -93,2 +95,5 @@ # DVO.GETSTAR.MAG.MAX F32 20.0 # XXX need to be able to limit the density! + +# XXX this does not really make sense -- a single chip model cannot be inconsistent in rotation with the model! +PSASTRO.ANGLE.TOLERANCE F32 10.0 Index: branches/eam_branches/20090820/ippconfig/vysos5/psphot.config =================================================================== --- branches/eam_branches/20090820/ippconfig/vysos5/psphot.config (revision 25206) +++ branches/eam_branches/20090820/ippconfig/vysos5/psphot.config (revision 25766) @@ -1,31 +1,16 @@ -SAVE.BACKMDL BOOL TRUE -SAVE.BACKGND BOOL FALSE -SAVE.BACKSUB BOOL FALSE -SAVE.PLOTS BOOL TRUE -SAVE.RESID BOOL FALSE -SAVE.PSF BOOL TRUE -LOAD.PSF BOOL FALSE +# XXX check on this value +PSF_CLUMP_NSIGMA F32 3.5 # region of Sx,Sy plane to use for selecting PSF stars -PEAKS_NSIGMA_LIMIT F32 20.0 # peak significance threshold -PEAKS_NSIGMA_LIMIT_2 F32 10.0 # peak significance threshold -PSF_MOMENTS_RADIUS F32 7 # calculate initial source moments with this radius -PSF_CLUMP_NSIGMA F32 3.5 # region of Sx,Sy plane to use for selecting PSF stars -PSF_MAX_NSTARS S32 400 # limit number of stars used for PSF model -MEASURE.APTREND BOOL TRUE - +# do we want to do 2-pass analysis? BREAK_POINT STR PASS1 +# keep these large since psfs are so undersampled PSPHOT.CR.NSIGMA.LIMIT F32 5.0 # sources with crNsigma greater that this get tagged as likely cosmic rays PSPHOT.EXT.NSIGMA.LIMIT F32 5.0 # sources with extNsigma greater that this get tagged as likely extended sources -USE_FOOTPRINTS BOOL TRUE # use new pmFootprint peak packaging -FOOTPRINT_NPIXMIN S32 5 # Minimum size of a pmFootprint -FOOTPRINT_NSIGMA_LIMIT F32 10 # threshold for bright pmFootprint detection -FOOTPRINT_NSIGMA_LIMIT_2 F32 10 # threshold for faint pmFootprint detection +# small psfs so small grow radius FOOTPRINT_GROW_RADIUS S32 1 # How much to grow bright footprints FOOTPRINT_GROW_RADIUS_2 S32 2 # How much to grow faint footprints -FOOTPRINT_CULL_NSIGMA_DELTA F32 4 # Cull peaks that aren't nsigma above coll to neighbour -FOOTPRINT_CULL_NSIGMA_MIN F32 1 # Minimum height of colls in units of skyStdev PSF_MODEL STR PS_MODEL_PGAUSS Index: branches/eam_branches/20090820/magic/README =================================================================== --- branches/eam_branches/20090820/magic/README (revision 25206) +++ (revision ) @@ -1,18 +1,0 @@ -To build the DetectStreaks portion of magic - -tar xf ~sydney/magic.tgz -tar xf ~sydney/ssa-core-cpp.tgz - -cd ssa-core-cpp -make distclean -./configure -make - -cd ../magic -make install - -# to build streaksremove and related tools - -cd ../remove/src -make install - Index: branches/eam_branches/20090820/magic/configure.tcsh =================================================================== --- branches/eam_branches/20090820/magic/configure.tcsh (revision 25206) +++ branches/eam_branches/20090820/magic/configure.tcsh (revision 25766) @@ -34,5 +34,4 @@ case --enable-profile case --pedantic - shift breaksw; # key/value options passed by build systems which we ignore Index: branches/eam_branches/20090820/magic/remove/src/streaksio.c =================================================================== --- branches/eam_branches/20090820/magic/remove/src/streaksio.c (revision 25206) +++ branches/eam_branches/20090820/magic/remove/src/streaksio.c (revision 25766) @@ -288,8 +288,8 @@ sfile->resolved_name = psStringCopy(sfile->pmfile->filename); + sfile->nHDU = psFitsGetSize(sfile->pmfile->fits); + // copy header from fpu sfile->header = (psMetadata*) psMemIncrRefCounter(sfile->pmfile->fpa->hdu->header); - - sfile->nHDU = psFitsGetSize(sfile->pmfile->fits); return sfile; @@ -654,4 +654,15 @@ streaksExit("", PS_EXIT_DATA_ERROR); } + + // Ensure input is of the expected type + psDataType expected = isMask ? PS_TYPE_IMAGE_MASK : PS_TYPE_F32; // Expected type for image + for (int i = 0; i < in->imagecube->n; i++) { + psImage *image = in->imagecube->data[i]; // Image of interest + if (image->type.type != expected) { + psImage *temp = psImageCopy(NULL, image, expected); + psFree(image); + in->imagecube->data[i] = temp; + } + } } setDataExtent(stage, in, (stage == IPP_STAGE_RAW) && !isMask); @@ -670,4 +681,5 @@ sfile->fits->options = psFitsOptionsAlloc(); sfile->fits->options->scaling = PS_FITS_SCALE_MANUAL; + sfile->fits->options->fuzz = false; sfile->fits->options->bitpix = bitpix; sfile->fits->options->bscale = bscale; @@ -1114,12 +1126,12 @@ // these gets are not necessary, we could just set the pixels to nan // but I want to get the counts - double imageVal = psImageGet(image, x, y); + double imageVal = image->data.F32[y][x]; psU32 maskVal; if (sfiles->stage == IPP_STAGE_RAW) { unsigned int xChip, yChip; cellToChipInt(&xChip, &yChip, sfiles->astrom, x, y); - maskVal = psImageGet(mask, xChip, yChip); + maskVal = mask->data.PS_TYPE_IMAGE_MASK_DATA[yChip][xChip]; } else { - maskVal = psImageGet(mask, x, y); + maskVal = mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x]; } if (maskVal & maskMask) { @@ -1127,11 +1139,11 @@ if (!isExciseValue(imageVal, sfiles->inImage->exciseValue)) { ++nandPixels; - psImageSet(image, x, y, exciseValue); + image->data.F32[y][x] = exciseValue; } if (weight) { - double weightVal = weight ? psImageGet(weight, x, y) : 0; + double weightVal = weight ? weight->data.F32[y][x] : 0; if (!isnan(weightVal)) { ++nandWeights; - psImageSet(weight, x, y, NAN); + weight->data.F32[y][x] = NAN; } } @@ -1212,13 +1224,10 @@ if (SFILE_IS_IMAGE(in)) { // Turn off compression (code adapted from pmFPAWrite) -#ifdef SAVE_AND_RESTORE_COMPRESSION int bitpix = out->fits->options ? out->fits->options->bitpix : 0; // Desired bits per pixel psFitsCompression *compress = psFitsCompressionGet(out->fits); // Current compression options -#endif if (!psFitsSetCompression(out->fits, PS_FITS_COMPRESS_NONE, NULL, 0, 0, 0)) { psError(PM_ERR_UNKNOWN, false, "failed to turn off compression for extension %d\n", extnum); streaksExit("", PS_EXIT_UNKNOWN_ERROR); } -#ifdef SAVE_AND_RESTORE_COMPRESSION if (out->fits->options) { out->fits->options->bitpix = 0; @@ -1235,5 +1244,4 @@ streaksExit("", PS_EXIT_UNKNOWN_ERROR); } -#endif } else { copyTable(out, in, extnum); Index: branches/eam_branches/20090820/magic/remove/src/streaksremove.c =================================================================== --- branches/eam_branches/20090820/magic/remove/src/streaksremove.c (revision 25206) +++ branches/eam_branches/20090820/magic/remove/src/streaksremove.c (revision 25766) @@ -12,10 +12,11 @@ static pmConfig *parseArguments(int argc, char **argv); static bool readAndCopyToOutput(streakFiles *sf, bool exciseAll); -static void exciseNonWarpedPixels(streakFiles *sfiles, double newMaskValue); +static void exciseNonWarpedPixels(streakFiles *sfiles, psImageMaskType newMaskValue); static bool warpedPixel(streakFiles *sfiles, int x, int y); -static void excisePixel(streakFiles *sfiles, unsigned int x, unsigned int y, bool streak, double newMaskValue); +static void excisePixel(streakFiles *sfiles, unsigned int x, unsigned int y, bool streak, psImageMaskType newMaskValue); static void writeImages(streakFiles *sf, bool exciseImageCube); static void updateAstrometry(streakFiles *sfiles); -static void censorSources(streakFiles *sfiles, psU32 maskStreak); +static void censorSources(streakFiles *sfiles, psImageMaskType maskStreak); +static long censorPixels(streakFiles *sfiles, psImage * pixels, bool checkNonWarpedPixels, psU16 maskStreak); int @@ -146,22 +147,7 @@ } - psTimerStart("REMOVE_STREAKS"); - for (int y=0 ; y < sfiles->inImage->numRows; y++) { - for (int x = 0; x < sfiles->inImage->numCols; x++) { - if (psImageGet(pixels, x, y)) { - ++totalStreakPixels; - if (!checkNonWarpedPixels || warpedPixel(sfiles, x, y)) { - - excisePixel(sfiles, x, y, true, maskStreak); - - } else { - // This pixel was not included in any warp and has thus already excised - // by exciseNonWarpedPixels - } - } - } - } + totalStreakPixels += censorPixels(sfiles, pixels, checkNonWarpedPixels, maskStreak); psLogMsg("streaksremove", PS_LOG_INFO, "time to remove streak pixels: %f\n", psTimerClear("REMOVE_STREAKS")); @@ -254,4 +240,27 @@ return 0; +} + +static long +censorPixels(streakFiles *sfiles, psImage *pixels, bool checkNonWarpedPixels, psU16 maskStreak) +{ + long streakPixels = 0; + + for (int y=0 ; y < sfiles->inImage->numRows; y++) { + for (int x = 0; x < sfiles->inImage->numCols; x++) { + if (psImageGet(pixels, x, y)) { + ++streakPixels; + if (!checkNonWarpedPixels || warpedPixel(sfiles, x, y)) { + + excisePixel(sfiles, x, y, true, maskStreak); + + } else { + // This pixel was not included in any warp and has thus already excised + // by exciseNonWarpedPixels + } + } + } + } + return streakPixels; } @@ -665,5 +674,5 @@ static void -excisePixel(streakFiles *sfiles, unsigned int x, unsigned int y, bool streak, double newMaskValue) +excisePixel(streakFiles *sfiles, unsigned int x, unsigned int y, bool streak, psImageMaskType newMaskValue) { double exciseValue = sfiles->inImage->exciseValue; @@ -674,17 +683,17 @@ } - double imageValue = psImageGet (sfiles->inImage->image, x, y); + float imageValue = sfiles->inImage->image->data.F32[y][x]; if (sfiles->recImage && !isExciseValue(imageValue, sfiles->inImage->exciseValue) ) { - psImageSet (sfiles->recImage->image, x, y, imageValue); + sfiles->recImage->image->data.F32[y][x] = imageValue; } if (sfiles->transparentStreaks == 0) { - psImageSet (sfiles->outImage->image, x, y, exciseValue); + sfiles->outImage->image->data.F32[y][x] = exciseValue; } else { if (streak) { // as a visualization aid don't mask the pixel, just change the intensity - psImageSet (sfiles->outImage->image, x, y, imageValue + sfiles->transparentStreaks); + sfiles->outImage->image->data.F32[y][x] = imageValue + sfiles->transparentStreaks; } else { - psImageSet (sfiles->outImage->image, x, y, exciseValue); + sfiles->outImage->image->data.F32[y][x] = exciseValue; } } @@ -692,21 +701,21 @@ if (sfiles->outWeight) { if (sfiles->recWeight) { - double weightValue = psImageGet (sfiles->inWeight->image, x, y); - psImageSet (sfiles->recWeight->image, x, y, weightValue); + sfiles->recWeight->image->data.F32[y][x] = sfiles->inWeight->image->data.F32[y][x]; } // Assume that weight images are always a floating point type - psImageSet (sfiles->outWeight->image, x, y, NAN); + sfiles->outWeight->image->data.F32[y][x] = NAN; } if (sfiles->outMask) { if (sfiles->recMask) { - double maskValue = psImageGet (sfiles->inMask->image, x, y); - psImageSet (sfiles->recMask->image, x, y, maskValue); - } - psImageSet (sfiles->outMask->image, x, y, newMaskValue); + sfiles->recMask->image->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = + sfiles->inMask->image->data.PS_TYPE_IMAGE_MASK_DATA[y][x]; + } + sfiles->outMask->image->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = + sfiles->inMask->image->data.PS_TYPE_IMAGE_MASK_DATA[y][x] | newMaskValue; } } static void -exciseNonWarpedPixels(streakFiles *sfiles, double newMaskValue) +exciseNonWarpedPixels(streakFiles *sfiles, psImageMaskType newMaskValue) { int cell_x0 = sfiles->astrom->cell_x0; @@ -784,5 +793,5 @@ // streak mask static void -censorSources(streakFiles *sfiles, psU32 maskStreak) +censorSources(streakFiles *sfiles, psImageMaskType maskStreak) { if ((!sfiles->inSources) || (!sfiles->outMask)) { @@ -855,5 +864,10 @@ psF32 y = psMetadataLookupF32(NULL, row, "Y_PSF"); - psU32 mask = psImageGet(maskImage, x, y); + psImageMaskType mask; + if ((x >= maskImage->numCols) || (y >= maskImage->numRows) || (x < 0) || (y < 0)) { + mask = maskStreak; + } else { + mask = maskImage->data.PS_TYPE_IMAGE_MASK_DATA[(int)y][(int)x]; + } // Key the source if the center pixel is not masked with maskStreak Index: branches/eam_branches/20090820/operations/detrend/.mana =================================================================== --- branches/eam_branches/20090820/operations/detrend/.mana (revision 25766) +++ branches/eam_branches/20090820/operations/detrend/.mana (revision 25766) @@ -0,0 +1,2 @@ +echo {600*4} +echo {datan(1/4)} Index: branches/eam_branches/20090820/operations/detrend/.pantasks =================================================================== --- branches/eam_branches/20090820/operations/detrend/.pantasks (revision 25766) +++ branches/eam_branches/20090820/operations/detrend/.pantasks (revision 25766) @@ -0,0 +1,659 @@ +status +server input input +setup +load.hosts wave.2 +load.hosts.wave.2 +load.hosts.wave2 +load.hosts.wave3 +run +status +add.label darktest.beaumont.1 +status +status +status +status +status +status +list.labels +show.labels +del.label darktest.beaumont.1 +status +shutdown now +server input input +setup +load.hosts.wave2 +load.hosts.wave3 +run +add.label beaumont.darktest.2 +status +show.labels +status +status +status +status +status +status +status +status +status +status +shutdown now +status +status +status +status +status +status +status +status +status +status +server input input +setup +load.hosts.wave2 +load.hosts.wave3 +run +status +add.label beaumont.darktest.2 +status +status +status +status +status +status +status +status +status +status +status +show.label +status +status +status +del.label beaumont.darktest.2 +show.label +ls +show.label +add.label dummy +status +show.label +status +status +status +add.label beaumont.darktest.3 +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +shutdown now +eixt +server input input +setup +load.hosts.wave2 +load.hosts.wave3 +run +status +status +status +status +shutdown now +ls +ls +server input input +setup +load.hosts.wave2 +load.hosts.wave3 +run +status +status +status +status +status +shutdown now +status +server input input +server setup +setup +load.hosts.wave2 +load.hosts.wave3 +run +status +status +shutdown now +status +server input input +setup +load.hosts.wave2 +run +status +status +status +shutdown now +input input +setup +load.hosts.wave2 +run +status +status +status +status +status +status +server input input +setup +load.hosts.wave2 +run +status +status +status +shutdown now +status +server input input +setup +load.wave.2 +load.hosts.wave.2 +load.hosts.wave2 +load.hosts.wave3 +status +status +run +status +status +shutdown now +status +status +server input input +setup +load.hosts.wave2 +load.hosts.wave3 +run +status +status +status +status +flatcorr.off +status +status +status +status +status +status +status +status +stat +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +shutdown now +status +server input input +setup +status +flatcorr.off +status +load.hosts.wave2 +load.hosts.wave3 +run +status +status +status +status +status +status +status +status +control status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +shutdown now +status +server input input +setup +status +load.hosts.wave2 +load.hosts.wave3 +load.hosts.compute +run +status +status +status -tasks +stop +status +status +status -taskstatsreset +status +run +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -taskstats +statu +status -tasks +echo {60*2} +stop +echo {60*2} +status -tasks +status -tasks +control status +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +control status +status +status +status +control status +status +status +status +status +status +run +status +status +status +status +status -tasks +status -tasks +status +status +status +status +status +status +status +control status +status +stop +status -taskstats +run +status +status +status +status +!dettool -revertstacked -dbname gpc1 -det_id 221 +status +status +status +status +status +!dettool -revertresidimfile -dbname gpc1 -det_id 221 +status +!dettool -revertstacked -dbname gpc1 -det_id 221 +status +status -taskstats +status +!dettool -revertresidimfile -dbname gpc1 -det_id 221 +status +!dettool -revertresidexp -dbname gpc1 -det_id 221 +!dettool -revertprocessedexp -dbname gpc1 -det_id 221 +status +status +status +status +status +status +status -tasks +status -tasks +status -task +status +stop +run +status +status +status -taskstats +status -taskstats +status +control status +control host off ipp049 +control status +server input input +setup +status +load.hosts.wave2 +load.hosts.wave3 +run +status +status +status +status -tasks +status +status -tasks +status +shutdown now +server input input +setup +status +load.hosts.wave2 +load.hosts.wave3 +load.hosts.compute +status +control status +run +status +status -tasks +!dettool -revertprocessedimfile -det_id 223 -dbname gpc1 +!dettool -revertprocessedexp -det_id 223 -dbname gpc1 +status +control status +status +status +control status +control status +control status +status +status -taskstats +status -taskstats +control status +control status +status +status +status +status -taskstats +!dettool -dbname gpc1 -revertresidimfile -det_id 223 +!dettool -dbname gpc1 -revertstacked -det_id 223 +status +status +status +status +status +status +status -tasks +status -tasks +!dettool -dbname gpc1 -revertresidexp -det_id 223 +status +shutdown now +server input iput +server input input +setup +status +load.hosts.wave2 +load.hosts.compute +status +control status +run +control status +control host off ipp026 +control status +status +status +status +control status +control status +status +shutdown onw +shutdown now +server input input +setup +status +load.hosts.wave2 +load.hosts.wave3 +load.hosts.compute +status +run +status +status -tasks +status -tasks +control status +status -tasks +status -tasks +status -tasks +status -tasks +!dettool -dbname gpc1 -revertprocessedimfile -det_id 224 +status -tasks +status -tasks +status -tasks +!dettool -dbname gpc1 -revertprocessedimfile -det_id 224 +status -tasks +!dettool -dbname gpc1 -revertprocessedexp -det_id 224 +status -tasks +status -taskstats +control status +control status +status -tasks +status +status -tasks +status -tasks +status -taskstats +!dettool -dbname gpc1 -revertresidimfile -det_id 224 +status -tasks +control status +status -tasks +!dettool -dbname gpc1 -revertresidimfile -det_id 224 +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status +status -taskstats +status -tasks +!dettool -dbname gpc1 -revertprocessedimfile -det_id 225 +!dettool -dbname gpc1 -revertprocessedexp -det_id 225 +status -task +status -task +status -task +status -tasks +!dettool -dbname gpc1 -revertprocessedexp -det_id 225 +!dettool -dbname gpc1 -revertstacked -det_id 225 +!dettool -dbname gpc1 -revertresidimfile -det_id 225 +status -tasks +status +status +status +stop +shutdown now +server input input +setup +status +load.hosts.wave3 +load.hosts.wave2 +load.hosts.compute +status +run +status +status +status +status +status +status -tasks +status -tasks +status -tasks +echo {50*60} +!dettool -dbname gpc1 -revertprocessedimfile -det_id 226 +!dettool -dbname gpc1 -revertprocessedexp -det_id 226 +status -tasks +status -tasks +!dettool -dbname gpc1 -revertprocessedimfile -det_id 226 +!dettool -dbname gpc1 -revertprocessedexp -det_id 226 +status +!dettool -dbname gpc1 -revertprocessedexp -det_id 226 +!ls +!ls -lrt +status +!dettool -dbname gpc1 -revertprocessedimfile -det_id 226 +!dettool -dbname gpc1 -revertstacked -det_id 226 +status +status +status +status +status +status +!dettool -dbname gpc1 -revertstacked -det_id 226 +status +!dettool -dbname gpc1 -revertresidimfile -det_id 226 +status +status +status +status +status +status +!dettool -dbname gpc1 -revertstacked -det_id 226 +status +status +status +status +status -tasks +status -tasks +!dettool -dbname gpc1 -revertresidimfile -det_id 226 +status -task +status -task +!dettool -dbname gpc1 -revertnorm -det_id 226 +!dettool -dbname gpc1 -revertnormalizedimfile -det_id 226 +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status +show.label +status +status -tasks +shutdown now +status +server input input +setup +load.hosts.wave2 +load.hosts.wave3 +load.hosts.compute +run +status +status +status -tasks +status -tasks +status -tasks +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +show.labels +shutdown now Index: branches/eam_branches/20090820/operations/detrend/dark.20090712.sh =================================================================== --- branches/eam_branches/20090820/operations/detrend/dark.20090712.sh (revision 25766) +++ branches/eam_branches/20090820/operations/detrend/dark.20090712.sh (revision 25766) @@ -0,0 +1,18 @@ +#!/bin/csh -f + +# 20090712 +# attempting a dark analysis using a range of temperatures. + +dettool -pretend -dbname gpc1 -definebyquery \ + -det_type DARKTEST \ + -workdir neb://@HOST@.0/gpc1/detrend.20090712 \ + -inst GPC1 \ + -select_ccd_temp_min -76 \ + -select_pon_time_min 160000 \ + -select_dateobs_begin 2009-04-01T00:00:00 \ + -select_dateobs_end 2009-07-15T10:00:00 \ + -select_exp_type dark \ + -comment 'darks_%' \ + -use_begin 2009-01-01 \ + -random_subset \ + -random_limit 41 Index: branches/eam_branches/20090820/operations/detrend/dark.20090713.sh =================================================================== --- branches/eam_branches/20090820/operations/detrend/dark.20090713.sh (revision 25766) +++ branches/eam_branches/20090820/operations/detrend/dark.20090713.sh (revision 25766) @@ -0,0 +1,18 @@ +#!/bin/csh -f + +# 20090713 +# attempting a dark analysis using a range of temperatures. + +dettool -dbname gpc1 -definebyquery \ + -det_type DARKTEST \ + -workdir neb://@HOST@.0/gpc1/detrend.20090712 \ + -inst GPC1 \ + -select_ccd_temp_min -76 \ + -select_pon_time_min 160000 \ + -select_dateobs_begin 2009-04-01T00:00:00 \ + -select_dateobs_end 2009-07-15T10:00:00 \ + -select_exp_type dark \ + -comment 'darks_%' \ + -use_begin 2009-01-01T00:00:00 \ + -random_subset \ + -random_limit 60 Index: branches/eam_branches/20090820/operations/detrend/dark.20090714.sh =================================================================== --- branches/eam_branches/20090820/operations/detrend/dark.20090714.sh (revision 25766) +++ branches/eam_branches/20090820/operations/detrend/dark.20090714.sh (revision 25766) @@ -0,0 +1,19 @@ +#!/bin/csh -f + +# 20090714 +# attempting a dark analysis using a smaller range of temperatures. + +dettool -dbname gpc1 -definebyquery \ + -det_type DARKTEST \ + -workdir neb://@HOST@.0/gpc1/detrend.20090712 \ + -inst GPC1 \ + -select_ccd_temp_min -60 \ + -select_pon_time_min 160000 \ + -select_dateobs_begin 2009-04-01T00:00:00 \ + -select_dateobs_end 2009-07-15T10:00:00 \ + -select_exp_type dark \ + -comment 'darks_%' \ + -use_begin 2009-01-01T00:00:00 \ + -time_begin 2009-01-01T00:00:00 \ + -random_subset \ + -random_limit 60 Index: branches/eam_branches/20090820/operations/detrend/dark.20090716.sh =================================================================== --- branches/eam_branches/20090820/operations/detrend/dark.20090716.sh (revision 25766) +++ branches/eam_branches/20090820/operations/detrend/dark.20090716.sh (revision 25766) @@ -0,0 +1,22 @@ +#!/bin/csh -f + +# 20090716 : EAM : It turns out that the camera reports bogus DARKTIME +# value occasionally. I've changed gpc1/ppMerge.config to use the +# EXPTIME instead, and initial test show that does a much better job. +# Here I am re-attempting a dark analysis using the full range of +# temperatures. + +dettool -dbname gpc1 -definebyquery \ + -det_type DARKTEST \ + -workdir neb://@HOST@.0/gpc1/detrend.20090716 \ + -inst GPC1 \ + -select_ccd_temp_min -75 \ + -select_pon_time_min 160000 \ + -select_dateobs_begin 2009-04-01T00:00:00 \ + -select_dateobs_end 2009-07-15T10:00:00 \ + -select_exp_type dark \ + -comment 'darks_%' \ + -use_begin 2009-01-01T00:00:00 \ + -time_begin 2009-01-01T00:00:00 \ + -random_subset \ + -random_limit 60 Index: branches/eam_branches/20090820/operations/detrend/fringe.20090717.sh =================================================================== --- branches/eam_branches/20090820/operations/detrend/fringe.20090717.sh (revision 25766) +++ branches/eam_branches/20090820/operations/detrend/fringe.20090717.sh (revision 25766) @@ -0,0 +1,23 @@ +#!/bin/csh -f + +# 20090717 : EAM : Generate a fringe frame selecting y-band images with the sun down > 30 degrees + +dettool -simple -dbname gpc1 -definebyquery \ + -det_type FRINGE \ + -workdir neb://@HOST@.0/gpc1/detrend.20090717 \ + -inst GPC1 \ + -select_pon_time_min 200000 \ + -select_dateobs_begin 2009-04-01T00:00:00 \ + -select_dateobs_end 2009-07-15T10:00:00 \ + -select_exp_type object \ + -select_exp_time_min 200.0 \ + -select_filter y.00000 \ + -select_sun_alt_max -30 \ + -filter y \ + -select_exp_type object \ + -comment 'MD%' \ + -use_begin 2009-01-01T00:00:00 \ + -time_begin 2009-01-01T00:00:00 \ + -random_subset \ + -random_limit 60 + Index: branches/eam_branches/20090820/operations/detrend/fringe.20090722.sh =================================================================== --- branches/eam_branches/20090820/operations/detrend/fringe.20090722.sh (revision 25766) +++ branches/eam_branches/20090820/operations/detrend/fringe.20090722.sh (revision 25766) @@ -0,0 +1,25 @@ +#!/bin/csh -f + +echo "done" +exit 1 +# 20090722 : EAM : Generate a fringe frame selecting y-band images with the sun down > 30 degrees and moon phase > 0.75 + +dettool -simple -dbname gpc1 -definebyquery \ + -det_type FRINGE \ + -workdir neb://@HOST@.0/gpc1/detrend.20090717 \ + -inst GPC1 \ + -select_pon_time_min 200000 \ + -select_dateobs_begin 2009-04-01T00:00:00 \ + -select_dateobs_end 2009-07-22T10:00:00 \ + -select_exp_type object \ + -select_exp_time_min 25.0 \ + -select_filter y.00000 \ + -select_sun_alt_max -30 \ + -select_sun_alt_max -30 \ + -select_moon_phase_min 0.75 \ + -comment 'ThreePi%' \ + -filter y \ + -use_begin 2009-01-01T00:00:00 \ + -time_begin 2009-01-01T00:00:00 \ + -random_subset \ + -random_limit 50 Index: branches/eam_branches/20090820/operations/detrend/input =================================================================== --- branches/eam_branches/20090820/operations/detrend/input (revision 25766) +++ branches/eam_branches/20090820/operations/detrend/input (revision 25766) @@ -0,0 +1,133 @@ + +macro setup + module pantasks.pro + detrend.modules + + module site.mhpcc.pro + + init.copy.mhpcc on + queueload tmp -x "cat $MODULES:0/ipphosts.mhpcc.config" + ipptool2book tmp ipphosts -key camera + + # XXX merge the above into init.site more cleanly (careful with summit copy stuff) + # init.site (we do not use init.site because it loads the hosts) + + add.database gpc1 + set.poll 100 +end + +macro set.poll + if ($0 != 2) + echo "USAGE:set.poll (value)" + break + end + + $POLLLIMIT = $1 +end + +macro get.poll + echo "poll limit: $POLLLIMIT" +end + +macro load.hosts.wave1 + controller host add ipp005 -threads 4 + controller host add ipp006 -threads 4 +# controller host add ipp007 -threads 4 (respawning?) +# controller host add ipp008 -threads 4 (flaky) + controller host add ipp009 -threads 4 + controller host add ipp010 -threads 4 + controller host add ipp011 -threads 4 + controller host add ipp012 -threads 4 + controller host add ipp013 -threads 4 +# controller host add ipp014 -threads 4 +# controller host add ipp016 -threads 4 (flaky) + controller host add ipp017 -threads 4 +# controller host add ipp018 -threads 4 (flaky) +# controller host add ipp019 -threads 4 (nebulous) + controller host add ipp020 -threads 4 + controller host add ipp021 -threads 4 +end + +macro load.hosts.wave2 + controller host add ipp015 -threads 4 + controller host add ipp017 -threads 4 + controller host add ipp023 -threads 4 + controller host add ipp024 -threads 4 + controller host add ipp026 -threads 4 + controller host add ipp027 -threads 4 + controller host add ipp028 -threads 4 + controller host add ipp029 -threads 4 + controller host add ipp030 -threads 4 + controller host add ipp031 -threads 4 + controller host add ipp032 -threads 4 + controller host add ipp033 -threads 4 + controller host add ipp034 -threads 4 + controller host add ipp035 -threads 4 + controller host add ipp036 -threads 4 +end + +macro load.hosts.wave3 + controller host add ipp038 -threads 4 + controller host add ipp039 -threads 4 + controller host add ipp040 -threads 4 + controller host add ipp041 -threads 4 + controller host add ipp042 -threads 4 + controller host add ipp043 -threads 4 + controller host add ipp044 -threads 4 + controller host add ipp045 -threads 4 + controller host add ipp046 -threads 4 + controller host add ipp047 -threads 4 + controller host add ipp048 -threads 4 + controller host add ipp049 -threads 4 + controller host add ipp050 -threads 4 + controller host add ipp051 -threads 4 + controller host add ipp052 -threads 4 + controller host add ipp053 -threads 4 +end + +macro load.hosts.compute + controller host add ippc00 -threads 4 + controller host add ippc01 -threads 4 + controller host add ippc02 -threads 4 + controller host add ippc03 -threads 4 + controller host add ippc04 -threads 4 + controller host add ippc05 -threads 4 + controller host add ippc06 -threads 4 +# controller host add ippc07 -threads 4 (pantasks - register) + controller host add ippc08 -threads 4 + controller host add ippc09 -threads 4 +# controller host add ippc10 -threads 4 (pantasks - stdscience) + controller host add ippc11 -threads 4 + controller host add ippc12 -threads 4 + controller host add ippc13 -threads 4 + controller host add ippc14 -threads 4 + controller host add ippc15 -threads 4 + controller host add ippc16 -threads 4 + controller host add ippc17 -threads 4 + controller host add ippc18 -threads 4 + controller host add ippc19 -threads 4 +end + +macro show.book + if ($0 != 2) + echo "USAGE: show.book (book)" + break + end + + book npages $1 -var npages + for i 0 $npages + book getpage $1 $i -var pagename + book listpage $1 $pagename + end + + echo "npages: $npages" +end + +macro del.page.from.book + if ($0 != 3) + echo "USAGE: show.book (book) (page)" + break + end + + book delpage $1 $2 +end Index: branches/eam_branches/20090820/operations/detrend/ptolemy.rc =================================================================== --- branches/eam_branches/20090820/operations/detrend/ptolemy.rc (revision 25766) +++ branches/eam_branches/20090820/operations/detrend/ptolemy.rc (revision 25766) @@ -0,0 +1,5 @@ + +PANTASKS_SERVER_STDOUT pantasks.stdout.log +PANTASKS_SERVER_STDERR pantasks.stderr.log +PANTASKS_SERVER ippc06 +PASSWORD foobar Index: branches/eam_branches/20090820/operations/distribution/.pantasks =================================================================== --- branches/eam_branches/20090820/operations/distribution/.pantasks (revision 25766) +++ branches/eam_branches/20090820/operations/distribution/.pantasks (revision 25766) @@ -0,0 +1,843 @@ +server input input +status +run +status +status -tasks +status +status -tas +status -tasks +status -tasks +status +status -tasks +status -tasks +status +status -tasks +status -tasks +status +status -tasks +halt +run +stop +status +run +status +status +status +status +status +status -tasks +status -tasks +status +status -tasks +status +status -tasks +dist.status +status -tasks +status -tasks +status -tasks +status +status -tasks +status +status -tasks +status +status -tasks +status -tasks +status -tasks +show.labels +del.label MD06.200904.v1 +add.label MD09.200906.v1 +add.label MD09.200907.v1 +del.label MD07.200904.v1 +del.label MD08.200904.v1 +del.label MD08.200905.v1 +del.label MD08.200904.v1 +show.labels +del.label MD06.200905.v1 +del.label MD07.200905.v1 +status -tasks +status +controller status +status +status -tasks +dist.status +show.labels +status -tasks +status -tasks +status +status +status +status +show.labels +add.label MD06.200904.v1 +add.label MD06.200905.v1 +add.label MD07.200904.v1 +add.label MD07.200905.v1 +add.label MD08.200905.v1 +add.label MD08.200904.v1 +status -tasks +status +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status +status +statusquit +status +statua -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status +status -tasks +status -tasks +controller status +status -taskstats +status -tasks +status +status +dist.status +show.labels +add.label MD08.200906.v1 +status +status +status +status -tasks +status +status +status +status -tasks +status -tasks +status +status -tasks +status +status +status +status +status +status -tasks +status -tasks +status -tasks +status -tasks +status +status -tasks +status -tasks +status +status -tasks +halt +status -tasks +controller status +controller host off ipp053 +controller host off ipp053 +controller host delete ipp053 +controller host delete ipp053 +controller status +run +status +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status +status +status -tasks +status +status -tasks +status -tasks +status +status +status +controller status +controller status +dist.status +dist.status +stop +status +status +run +status +status +status +dist.off +status +status +status +controller tatus +controller status +run +status +status +status +status +status +status +status +status +dist.status +dist.reset +run +status +status +status +status +dist.on +status +status +status +status +status +controller status +status +status +status -tasks +status -tasks +status -tasks +status -tasks +status +controller status +controller status +status -tasks +status +controller status +stop +status +shutdown now +server input input +setup +run +status +status +show.labels +add.label MD06.200906.v1 +add.label MD07.200906.v1 +add.label MD08.200906.v1 +add.label MD09.200906.v1 +add.label MD09.200907.v1 +status -tasks +status +controller status +status -tasks +status +status +status +status +status +status +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status +status +status -tasks +status -tasks +status +status +status -tasks +status -tasks +status +status +status +status +status -tasks +status +status +status +status +status +status +status +shutdown now +server input input +dist.off +status +status +run +status +status +status +show.labels +show.labels +status +status -tastks +status -takss +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status +status +status +halt +server module dist.pro +shutdown now +server input input +server input input +del.label MD08.200904.v1 +del.label MD08.200905.v1 +del.label MD08.200906.v1 +add.label ThreePi_SouthernRegion.090724 +status +run +status +status +status -tasks +status -tasks +status -tasks +status -tasks +halt +module rcserver.pro +status +server module rcserver.pro +status +dist.off +status +run +status +status +halt +status +run +status +status +status +run +status +status +dist.on +status +status +status +status +status +status +status +status +status +status -tasks +status -tasks +status +status +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status +status -tasks +status -tasks +status +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status +status -tasks +status -tasks +status -tasks +status +status +status -taskstats +controller status +status -tasks +halt +run +status +run +status -tasks +status -tasks +status -tasks +status -tasks +status +status -tasks +status -tasks +status +status +status +status +status -tasks +status -tasks +status -takss +status -tasks +status -tasks +status -tasks +status +dist.off +status +status +status +status +status +status +status +status +status +dist.on +status -tasks +status -tasks +status -tasks +status +status -tasks +status +controller status +status +status -tasks +status -tasks +status -tasks +status +status -tasks +status +status -taskstats +status -tasks +status -tasks +status -tasks +status -tasks +controller status +status -tasks +dist.off +status +status -tasks +status -tasks +status -tasks +stop +status -tasks +status +status +status +status +status +status +status +run +status +run +status +dist.on +status +status +status +dist.off +status +status +status +dist.on +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +controller status +status +stop +run +status +status +status +status +status +status +status +status +status +stop +status -tasks +status +!pgrep pclient | wc +shutdown now +status +server input input +shutdown now +server input input +shutdown now +server input input +status +run +run +status +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +halt +server module rcserver.pro +status +status +run +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status +status +status +status +status -tasks +status +status -tasks +status -tasks +status -tasks +status +status +status +status +status -tasks +status -tasks +status +status +status -tasks +status -tasks +status -tasks +status -tasks +status -taskstats +status -tasks +status +status +status -tasks +status -tasks +status +add.label MD09.200906.v1 +status +status -tasks +status -tasks +status -tasks +status +show.labels +add.label MD09.200907.v1 +show.labels +status +status +!hostname +status +shutdown now +server input input +status +run +status +status +status -tasks +status -tasks +status -tasks ; !date +status -tasks ; !date +status -tasks ; !date +show.labels +show.labels +show.labels +status -tasks ; !date +status -tasks ; !date +status -tasks +server module rcserver +server module rcserver.pro +status +status +status +status +status -tasks +status +status +status +status +status +status +status +status +status +status -tasks +status +status +status +status +status +status +status +status -tasks +status +status -tasks +status +status -tasks +!hostname +status -tasks +stattus +status +status +status -tasks +status -tasks +si 0 +status +status -tasks +status -tasks +status -tasks +halt +status +run +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status +status +status +status +status +status +status +status +controller status; +status +status +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status +status +status -tasks +status +add.label ThreePi_SouthernRegion.090724 +status +status +status +status +status +controller status +status +status -tasks +status -tasks +status -tasks +status +status t-asks +show.labels +show.labels +status -takss +status -tasks +halt +run +stop +shutdown now +server input input +run +status +status +status -tasks +status +status -tasks +status +status +status -tasks +status -tasks +status -tasks +status -tasks +status +status +status +stop +run +status +status +status -taskststats +status -taskstats +server input /data/ipp004.0/home/bills/bump +status +status +status +status +status +status +status +status +status -taskstats +status -taskstats +status +status +status +status +stop +server module rcserver.pro +run +status +status +status +status +status +status +status +status +status +status +status +status +controller status +controller status +status -tasks +status -tasks +status -tasks +status -taskstats +status -taskstats +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +status +status +status +status -taskstats +status +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +controller status +show.labels +stop +status +status +status +ex +status +status +status +status +status +halt +runt +run +status +status +status -tasks +status +status -tasks +status -tasks +status +status +status +status +status -tasks +status -tasks +status -tasks +status -tasks +status -tasks +dist.off +rcserver.off +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +status +shutdown now Index: branches/eam_branches/20090820/operations/distribution/input =================================================================== --- branches/eam_branches/20090820/operations/distribution/input (revision 25766) +++ branches/eam_branches/20090820/operations/distribution/input (revision 25766) @@ -0,0 +1,71 @@ +module pantasks.pro + +module site.mhpcc.pro +init.copy.mhpcc on +queueload tmp -x "cat $MODULES:0/ipphosts.mhpcc.config" +ipptool2book tmp ipphosts -key camera + +module dist.pro + +#module rcserver.pro + +add.database gpc1 + +macro add.hosts + controller host add ipp021 -threads 4 + controller host add ipp024 -threads 4 + controller host add ipp026 -threads 4 + controller host add ipp028 -threads 4 + controller host add ipp029 -threads 4 + controller host add ipp030 -threads 4 + controller host add ipp031 -threads 4 + controller host add ipp032 -threads 4 + controller host add ipp033 -threads 4 + controller host add ipp034 -threads 4 + controller host add ipp035 -threads 4 + controller host add ipp036 -threads 4 + controller host add ipp038 + controller host add ipp039 + controller host add ipp040 + controller host add ipp041 + controller host add ipp042 + controller host add ipp043 + controller host add ipp044 + controller host add ipp045 + controller host add ipp046 + controller host add ipp047 + controller host add ipp048 + controller host add ipp049 + controller host add ipp050 + controller host add ipp051 + controller host add ipp052 + controller host add ipp053 +end + +macro skycell.hosts + controller host add ipp024 + controller host add ipp013 + controller host add ipp026 + controller host add ipp028 + controller host add ipp029 + controller host add ipp030 + controller host add ipp031 + controller host add ipp032 + controller host add ipp033 + controller host add ipp034 + controller host add ipp035 + controller host add ipp036 +end + +skycell.hosts +#add.hosts + +#add.label MD08.200904.v1 +#add.label MD08.200905.v1 +#add.label MD08.200906.v1 +#add.label ThreePi_SouthernRegion.090724 +add.label STS.090724 + +echo labels: +show.labels + Index: branches/eam_branches/20090820/operations/distribution/ptolemy.rc =================================================================== --- branches/eam_branches/20090820/operations/distribution/ptolemy.rc (revision 25766) +++ branches/eam_branches/20090820/operations/distribution/ptolemy.rc (revision 25766) @@ -0,0 +1,5 @@ + +PANTASKS_SERVER_STDOUT pantasks.stdout.log +PANTASKS_SERVER_STDERR pantasks.stderr.log +PANTASKS_SERVER ippc02 +PASSWORD foobar Index: branches/eam_branches/20090820/operations/home/.alias =================================================================== --- branches/eam_branches/20090820/operations/home/.alias (revision 25766) +++ branches/eam_branches/20090820/operations/home/.alias (revision 25766) @@ -0,0 +1,27 @@ +### Aliases + +alias . pwd +#alias .. cd .. +alias - cd - +alias / cd / +alias more less +alias h "history | tail" +alias ff 'find . -name \!* -print' +# alias fg 'find . -name \!#:1 -exec grep -H \!#:2 {} \;' +alias df df -k +alias ls ls -F --color=auto +alias ll ls -l +alias quota quota -v +alias mv mv -i +alias cp cp -i +alias purge 'rm -f #*# *~ ' +alias iraf "xgterm -sb -sl 400 -bg white -fg black -geometry 80x24+10+30 -title IRAF -fn -adobe-courier-bold-r-normal--14-140-75-75-m-90-iso8859-1 -e /home/mithrandir/price/iraf/runiraf & " +#alias loads 'rup maestro majestic magus msokcf misfit mizar mork mindy marbles miniskirt magellan mistress | sort -k 8 -n' +alias rsync rsync -e ssh +alias text2ps 'a2ps --columns=1 -L 70 -o \!#:2 \!#:1' +alias ds9 ds9 -zscale +alias make make -j4 +alias grep grep --colour +alias cvs cvs -q +alias lte libtool --mode=execute +alias mmi 'make && make install' Index: branches/eam_branches/20090820/operations/home/.bindkey =================================================================== --- branches/eam_branches/20090820/operations/home/.bindkey (revision 25766) +++ branches/eam_branches/20090820/operations/home/.bindkey (revision 25766) @@ -0,0 +1,131 @@ +#bindkey -k up history-search-backward # Up +#bindkey -k down history-search-forward # Down +bindkey '\e[1~' beginning-of-line # Home +#bindkey '\e[2~' dabbrev-expand # Insert +bindkey '\e[2~' overwrite-mode # Insert +bindkey '\e[3~' delete-char # Del +bindkey '\e[4~' end-of-line # End +bindkey '\e[5~' history-search-backward # Page up: search history backwards for line +#bindkey '\e[5~' complete-word-back # page up +bindkey '\e[6~' history-search-forward # Page down: search history forwards for line +#bindkey '\e[6~' complete-word-fwd # page down +bindkey '\e[20~' complete-word-fwd # F9 +bindkey '\e[21~' complete-word-back # F10 +bindkey ^[[3D backward-word # Alt-left arrow +bindkey ^[[3C forward-word # Alt-right arrow +bindkey -b M-\b backward-delete-word # Meta-backspace + +bindkey -b M-s i-search-fwd +bindkey -b M-r i-search-back +bindkey -b M-$ expand-variables +#bindkey '\244' expand-variables # M-$ notation seems to be broken +bindkey -b M-* expand-glob +bindkey -b M-8 expand-glob +#bindkey '\e8' expand-glob +bindkey -b M-4 expand-variables +#bindkey '\e4' expand-variables +bindkey -b M-_ insert-last-word +bindkey ' ' magic-space + +bindkey -b M-b backward-word +bindkey -b M-c capitalize-word +bindkey -b M-d delete-word +bindkey -b M-f forward-word +bindkey -b M-h run-help +bindkey -b M-l downcase-word +bindkey -b M-n history-search-forward +bindkey -b M-p history-search-backward +bindkey -b M-r toggle-literal-history +bindkey -b M-u upcase-word +bindkey -b M-w copy-region-as-kill +bindkey -b M-y yank # Yank cut text +bindkey -b C-space set-mark-command # Ctrl-space: mark text + +### Don't work, or I don't know what they do. +#bindkey '\e[7~' beginning-of-line # rxvt home +#bindkey '\e[8~' end-of-line # rxvt end +#bindkey '\e[\304' backward-word # rxvt M-left +#bindkey '\e[\303' forward-word # rxvt M-right +#bindkey '\e[\301' beginning-of-line # rxvt M-up +#bindkey '\e[\302' end-of-line # rxvt M-down +#bindkey '\eOd' backward-word # rxvt C-left +#bindkey '\eOc' forward-word # rxvt C-right +#bindkey '\eOa' upcase-word # rxvt C-up +#bindkey '\eOb' downcase-word # rxvt C-down +#bindkey '\e[3\376' delete-word # rxvt M-delete + + +### ESC-left-arrow : go to beginning of left word. +### The second version is used to fix a strange bug where the binding +### stops working after some usage. Did not manage to recreate. +#bindkey ^[^[[D vi-word-back +#bindkey \x{4420}vi-word-back + +### ESC-right-arrow : go to beginning of right word. +### The second version is used to fix a strange bug where the binding +### stops working after some usage. Did not manage to recreate. +#bindkey ^[^[[C vi-word-fwd +#bindkey \x{4320}vi-word-fwd + +### F1 : help on command currently typed(if 'ls passwd', help on 'ls'). +### first: while in console mode, second: while in X +#bindkey ^[[[A run-help +#bindkey \x{5020} run-help + +### F2 : set the mark command to cursor position. +### first: while in console mode, second: while in X +#bindkey ^[[[B set-mark-command +#bindkey \x{5120} set-mark-command + +### F3 : move cursor to the marked position. +### first: while in console mode, second: while in X +#bindkey ^[[[C exchange-point-and-mark +#bindkey \x{5220} exchange-point-and-mark + +### F4 : --empty-- +### first: while in console mode, second: while in X +#bindkey ^[[[D undefined-key +#bindkey \x{5320} undefined-key + +### F5 : check line for spelling and make changes. +### first: while in console mode, second: while in X +#bindkey ^[[[E spell-line +#bindkey [15~ spell-line + +### F6 : check current word for spelling and make changes. +### same in both console and X modes +#bindkey ^[[17~ spell-word + +### F7 : insert last item of previous command. +#bindkey ^[[18~ insert-last-word + +### F8 : search in history backwards for line beginning as current. +#bindkey ^[[19~ history-search-backward + +### F9 : clear screen. +### You may be in the middle of a command when you use this. +### Does not affect what you are writing at the moment. +#bindkey ^[[20~ clear-screen + +### F10 : do an 'ls -l'. (\16 is Ctrl-U on Linux(and Sun?)) +#bindkey -s ^[[21~ "\16ls -l\n" + +### F11 : display load average and current process status. +#bindkey ^[[23~ "/usr/bin/uptime ; ps" + +### F12 : do a ala-csh completion. +#bindkey ^[[24~ complete-word-raw + +### Thanks to Carlos Duarte +### Eazy edit of path, type Ctrl-X p +bindkey -s '^Xp' '. `echo $path`^X*)^A^Dset path = ( ' + +### Ctrl-X * Expand glob. example: ls *<^X*> will expand the line +bindkey -s ^[[21~ "\16ls -l\n" + +### F11 : display load average and current process status. +#bindkey ^[[23~ "/usr/bin/uptime ; ps" + +### F12 : do a ala-csh completion. +#bindkey ^[[24~ complete-word-raw + Index: branches/eam_branches/20090820/operations/home/.complete =================================================================== --- branches/eam_branches/20090820/operations/home/.complete (revision 25766) +++ branches/eam_branches/20090820/operations/home/.complete (revision 25766) @@ -0,0 +1,507 @@ +### From Michael Schroeder + # updates by John Gotts + # these from Tom Warzeka + + + +### This allows you to type "slogin price@", hit TAB and get a list of options +set userhosts = (price@mithrandir.ifa.hawaii.edu price@hubble.ifa.hawaii.edu dph8pap@stargate.dur.ac.uk ) +set hostnames = `cat $HOME/.ssh/known_hosts | cut -f 1 -d ' ' | cut -f 1 -d ,` >& /dev/null +set hosts = ( $userhosts $hostnames ) + +#complete ssh 'p/*/$hosts/' +complete ssh 'p/1/$hosts/' 'c/-/(l n)/' 'n/-l/u/' 'N/-l/c/' 'n/-/c/' 'p/2/c/' 'p/*/f/' +#complete slogin 'p/*/$hosts/' +complete slogin 'p/1/$hosts/' 'c/-/(l 8 e)/' 'n/-l/u/' +#complete scp 'p/1/f/' 'p/*/$hosts/:/' +### This one will rsh to the file to fetch the list of files! +complete scp 'p/1/f/' 'c%*@*:%`set filesonhost=$:-0;set filesonhost="$filesonhost:s/@/ /";set filesonhost="$filesonhost:s/:/ /";set filesonhost=($filesonhost " ");ssh $filesonhost[2] -l $filesonhost[1] ls -dp $filesonhost[3]\*`%' 'c%*:%`set filesonhost=$:-0;set filesonhost="$filesonhost:s/:/ /";set filesonhost=($filesonhost " ");ssh $filesonhost[1] ls -dp $filesonhost[2]\*`%' 'c%*@%$hosts%:' 'C@[./$~]*@f@' 'n/*/$hosts/:/' + +complete finger 'c/*@/$hosts/' 'n/*/u/@' +complete ping 'p/1/$hosts/' +complete traceroute 'p/1/$hosts/' + +complete ftp 'c/-/(d i g n v)/' 'n/-/$hosts/' 'p/1/$hosts/' 'n/*/n/' + +### Directory completion +complete {c,push,p}d 'p/1/d/' + +### Common commands +complete man 'p/*/c/' +complete which 'p/*/c/' +complete where 'p/*/c/' +complete printenv 'n/*/e/' +complete setenv 'p/1/e/' 'c/*:/f/' +complete set 'c/*=/f/' 'p/1/s/=' 'n/=/f/' +complete unset 'n/*/s/' +complete unalias 'n/*/a/' +complete limit 'c/-/(h)/' 'n/*/l/' +complete unlimit 'c/-/(h)/' 'n/*/l/' +complete su 'c/--/(login fast preserve-environment command shell \ + help version)/' 'c/-/(f l m p c s -)/' \ + 'n/{-c,--command}/c/' \ + 'n@{-s,--shell}@`cat /etc/shells`@' 'n/*/u/' + +complete grep 'c/-*A/x:<#_lines_after>/' 'c/-*B/x:<#_lines_before>/' \ + 'c/--/(extended-regexp fixed-regexp basic-regexp \ + regexp file ignore-case word-regexp line-regexp \ + no-messages revert-match version help byte-offset \ + line-number with-filename no-filename quiet silent \ + text directories recursive files-without-match \ + files-with-matches count before-context after-context \ + context binary unix-byte-offsets)/' \ + 'c/-/(A a B b C c d E e F f G H h i L l n q r s U u V \ + v w x)/' \ + 'p/1/x:/' 'N/-*e/f/' \ + 'n/-*e/x:/' 'n/-*f/f/' 'n/*/f/' + +complete egrep 'c/-*A/x:<#_lines_after>/' 'c/-*B/x:<#_lines_before>/' \ + 'c/--/(extended-regexp fixed-regexp basic-regexp \ + regexp file ignore-case word-regexp line-regexp \ + no-messages revert-match version help byte-offset \ + line-number with-filename no-filename quiet silent \ + text directories recursive files-without-match \ + files-with-matches count before-context after-context \ + context binary unix-byte-offsets)/' \ + 'c/-/(A a B b C c d E e F f G H h i L l n q r s U u V \ + v w x)/' \ + 'p/1/x:/' 'N/-*e/f/' \ + 'n/-*e/x:/' 'n/-*f/f/' 'n/*/f/' + +complete fgrep 'c/-*A/x:<#_lines_after>/' 'c/-*B/x:<#_lines_before>/' \ + 'c/--/(extended-regexp fixed-regexp basic-regexp \ + regexp file ignore-case word-regexp line-regexp \ + no-messages revert-match version help byte-offset \ + line-number with-filename no-filename quiet silent \ + text directories recursive files-without-match \ + files-with-matches count before-context after-context \ + context binary unix-byte-offsets)/' \ + 'c/-/(A a B b C c d E e F f G H h i L l n q r s U u V \ + v w x)/' \ + 'p/1/x:/' 'N/-*e/f/' \ + 'n/-*e/x:/' 'n/-*f/f/' 'n/*/f/' + +complete sed 'c/--/(quiet silent version help expression file)/' \ + 'c/-/(n V e f -)/' 'n/{-e,--expression}/x: